diff --git a/bclib/__init__.py b/bclib/__init__.py index 3a60c6e..00a02cb 100644 --- a/bclib/__init__.py +++ b/bclib/__init__.py @@ -1 +1,29 @@ -__version__ = "3.34.2" +__version__ = "3.35.0" + +import bclib.context + +from bclib.listener.message import Message +from bclib.listener.message_type import MessageType +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + +from bclib.predicate import Predicate +from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders + +from bclib.db_manager import * + +# from bclib.context.client_source_context import ClientSourceContext +# from bclib.context.client_source_member_context import ClientSourceMemberContext +# from bclib.context.context import Context +# from bclib.context.restful_context import RESTfulContext +# from bclib.context.web_context import WebContext +# from bclib.context.request_context import RequestContext +# from bclib.context.rabbit_context import RabbitContext +# from bclib.context.socket_context import SocketContext +# from bclib.context.merge_type import MergeType +# from bclib.context.server_source_context import ServerSourceContext +# from bclib.context.server_source_member_context import ServerSourceMemberContext +# from bclib.context.source_context import SourceContext +# from bclib.context.source_member_context import SourceMemberContext +# from bclib.context.context_factory import ContextFactory +# from bclib.context.end_point_context import EndPointContext diff --git a/bclib/cache/__init__.py b/bclib/cache/__init__.py index 5409366..e69de29 100644 --- a/bclib/cache/__init__.py +++ b/bclib/cache/__init__.py @@ -1,2 +0,0 @@ -from bclib.cache.manager import CacheManager -from bclib.cache.factory import CacheFactory \ No newline at end of file diff --git a/bclib/cache/cache_item/function_cache_item.py b/bclib/cache/cache_item/function_cache_item.py index 40d9bd5..30f2c76 100644 --- a/bclib/cache/cache_item/function_cache_item.py +++ b/bclib/cache/cache_item/function_cache_item.py @@ -1,8 +1,9 @@ -from ..cache_item.base_cache_item import BaseCacheItem from typing import Callable +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class FunctionCacheItem(BaseCacheItem): - def __init__(self, data: "any", life_time:"int", function:"Callable") -> None: + def __init__(self, data: "any", life_time: "int", function: "Callable") -> None: super().__init__(data, life_time) self.__function = function @@ -11,4 +12,4 @@ def get_data(self, *args, **kwargs) -> "any": if data is None: data = self.__function(*args, **kwargs) self._update_data(data) - return data \ No newline at end of file + return data diff --git a/bclib/cache/cache_item/scalar_cache_item.py b/bclib/cache/cache_item/scalar_cache_item.py index 175bb47..1537d29 100644 --- a/bclib/cache/cache_item/scalar_cache_item.py +++ b/bclib/cache/cache_item/scalar_cache_item.py @@ -1,5 +1,6 @@ -from .base_cache_item import BaseCacheItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class ScalarCacheItem(BaseCacheItem): def __init__(self, data: "any", life_time: int) -> None: - super().__init__(data, life_time) \ No newline at end of file + super().__init__(data, life_time) diff --git a/bclib/cache/cache_manager.py b/bclib/cache/cache_manager.py new file mode 100644 index 0000000..330b862 --- /dev/null +++ b/bclib/cache/cache_manager.py @@ -0,0 +1,25 @@ +from abc import ABC, abstractmethod +from bclib.utility import DictEx +from bclib.cache.cache_status import CacheStatus + + +class CacheManager(ABC): + def __init__(self, options: "DictEx") -> None: + super().__init__() + self._options = options + + @abstractmethod + def cache_decorator(self, key: "str" = None, life_time: "int" = 0): ... + + @abstractmethod + def get_cache(self, key: "str") -> "list|any|None": ... + + @abstractmethod + def add_or_update(self, key: "str", data: "any", + life_time: "int" = 0) -> "CacheStatus": ... + + @abstractmethod + def clean(self) -> "CacheStatus": ... + + @abstractmethod + def reset(self, keys: "list[str]" = None) -> "CacheStatus": ... diff --git a/bclib/cache/factory.py b/bclib/cache/factory.py index 94fae8b..d267101 100644 --- a/bclib/cache/factory.py +++ b/bclib/cache/factory.py @@ -1,16 +1,18 @@ -from ..cache.manager import CacheManager +from cache.cache_manager import CacheManager from abc import ABC from bclib.utility import DictEx -from ..cache.in_memory_cache_manager import InMemoryCacheManager -from ..cache.no_cache import NoCacheManager +from bclib.cache.in_memory_cache_manager import InMemoryCacheManager +from bclib.cache.no_cache import NoCacheManager + class CacheFactory(ABC): @staticmethod - def create(options:"DictEx"=None) -> "CacheManager": - cache_type = str(options.type) if options is not None and options.has("type") else None + def create(options: "DictEx" = None) -> "CacheManager": + cache_type = str(options.type) if options is not None and options.has( + "type") else None if cache_type is not None: if cache_type == "memory": return InMemoryCacheManager(options) else: raise ValueError(f"Unknown type for cache ('${cache_type}')") - return NoCacheManager(options) \ No newline at end of file + return NoCacheManager(options) diff --git a/bclib/cache/in_memory_cache_manager.py b/bclib/cache/in_memory_cache_manager.py index 45e73a2..86e345f 100644 --- a/bclib/cache/in_memory_cache_manager.py +++ b/bclib/cache/in_memory_cache_manager.py @@ -1,23 +1,24 @@ from bclib.cache.cache_status import CacheStatus -from ..cache.signal_base_cache_manager import SignalBaseCacheManager +from bclib.cache.signal_base_cache_manager import SignalBaseCacheManager from bclib.utility import DictEx -from ..cache.cache_status import CacheStatus +from bclib.cache.cache_status import CacheStatus from typing import Callable -from ..cache.cache_item.base_cache_item import BaseCacheItem -from ..cache.cache_item.function_cache_item import FunctionCacheItem -from .cache_item.scalar_cache_item import ScalarCacheItem -from ..cache.value_item.base_value_item import BaseValueItem -from ..cache.value_item.array_value_item import ArrayValueItem -from ..cache.value_item.scalar_value_item import ScalarValueItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem +from bclib.cache.cache_item.function_cache_item import FunctionCacheItem +from bclib.cache.cache_item.scalar_cache_item import ScalarCacheItem +from bclib.cache.value_item.base_value_item import BaseValueItem +from bclib.cache.value_item.array_value_item import ArrayValueItem +from bclib.cache.value_item.scalar_value_item import ScalarValueItem from functools import wraps + class InMemoryCacheManager(SignalBaseCacheManager): - + def __init__(self, options: DictEx) -> None: super().__init__(options) - self.__cache_dict:"dict[str, BaseValueItem]" = dict() + self.__cache_dict: "dict[str, BaseValueItem]" = dict() - def __add_or_update(self, key:"str", cache_item:"BaseCacheItem", value_item:"BaseValueItem") -> "CacheStatus": + def __add_or_update(self, key: "str", cache_item: "BaseCacheItem", value_item: "BaseValueItem") -> "CacheStatus": if key not in self.__cache_dict: self.__cache_dict[key] = value_item(cache_item) return CacheStatus.ADDED @@ -29,7 +30,7 @@ def __add_or_update(self, key:"str", cache_item:"BaseCacheItem", value_item:"Bas print(repr(ex)) return CacheStatus.ERROR - def cache_decorator(self, key:"str"=None, life_time:"int"=0) -> "Callable": + def cache_decorator(self, key: "str" = None, life_time: "int" = 0) -> "Callable": """ Decorator that caches the result of a function for a specified key and life time. @@ -51,12 +52,12 @@ def decorator(function): @wraps(function) def wrapper(*args, **kwargs): - function_cache:"FunctionCacheItem" = function.cache + function_cache: "FunctionCacheItem" = function.cache return function_cache.get_data(*args, **kwargs) return wrapper return decorator - def reset(self, keys:"list[str]"=None) -> "CacheStatus": + def reset(self, keys: "list[str]" = None) -> "CacheStatus": """ Resets the cache by removing all items or specified keys. @@ -86,10 +87,10 @@ def clean(self) -> "CacheStatus": self.__cache_dict = cleaned_cache_dict return CacheStatus.CLEANED - def get_cache(self, key:"str") -> "list|any|None": + def get_cache(self, key: "str") -> "list|any|None": return self.__cache_dict[key].get_item() if key in self.__cache_dict else None - - def add_or_update(self, key: str, data: "any", life_time:"int"= 0) -> "CacheStatus": + + def add_or_update(self, key: str, data: "any", life_time: "int" = 0) -> "CacheStatus": """ Add or update an item in the cache. Args: diff --git a/bclib/cache/manager.py b/bclib/cache/manager.py deleted file mode 100644 index 363fd90..0000000 --- a/bclib/cache/manager.py +++ /dev/null @@ -1,23 +0,0 @@ -from abc import ABC, abstractmethod -from bclib.utility import DictEx -from ..cache.cache_status import CacheStatus - -class CacheManager(ABC): - def __init__(self, options:"DictEx") -> None: - super().__init__() - self._options = options - - @abstractmethod - def cache_decorator(self, key:"str"=None, life_time:"int"=0): ... - - @abstractmethod - def get_cache(self, key:"str") -> "list|any|None": ... - - @abstractmethod - def add_or_update(self, key:"str", data:"any", life_time:"int"=0) -> "CacheStatus": ... - - @abstractmethod - def clean(self) -> "CacheStatus": ... - - @abstractmethod - def reset(self, keys:"list[str]"=None) -> "CacheStatus": ... diff --git a/bclib/cache/no_cache.py b/bclib/cache/no_cache.py index 8ce35e8..76251d4 100644 --- a/bclib/cache/no_cache.py +++ b/bclib/cache/no_cache.py @@ -1,5 +1,5 @@ -from ..cache.manager import CacheManager -from ..cache.cache_status import CacheStatus +from cache.cache_manager import CacheManager +from bclib.cache.cache_status import CacheStatus class NoCacheManager(CacheManager): diff --git a/bclib/cache/signal_base_cache_manager.py b/bclib/cache/signal_base_cache_manager.py index fb04b17..f20320c 100644 --- a/bclib/cache/signal_base_cache_manager.py +++ b/bclib/cache/signal_base_cache_manager.py @@ -1,39 +1,46 @@ -from ..cache.manager import CacheManager +from cache.cache_manager import CacheManager from bclib.utility import DictEx -from ..cache.signaler.factory import SignalerFactory +from bclib.cache.signaler.factory import SignalerFactory import asyncio + class SignalBaseCacheManager(CacheManager): - DEFAULT_CLEAN_INTERVAL = 43200 #Seconds => 12 Hours; 0 for indefinitely - DEFAULT_RESET_INTERVAL = 86400 #Seconds => 24 Hours; 0 for indefinitely + DEFAULT_CLEAN_INTERVAL = 43200 # Seconds => 12 Hours; 0 for indefinitely + DEFAULT_RESET_INTERVAL = 86400 # Seconds => 24 Hours; 0 for indefinitely def __init__(self, options: "DictEx") -> None: super().__init__(options) - self.__clean_interval = int(self._options.clean_interval) if self._options.has("clean_interval") else SignalBaseCacheManager.DEFAULT_CLEAN_INTERVAL + self.__clean_interval = int(self._options.clean_interval) if self._options.has( + "clean_interval") else SignalBaseCacheManager.DEFAULT_CLEAN_INTERVAL if self.__clean_interval < 0: raise ValueError("Invalid input for clean_interval!") - self.__reset_interval = int(self._options.reset_interval) if self._options.has("reset_interval") else SignalBaseCacheManager.DEFAULT_RESET_INTERVAL + self.__reset_interval = int(self._options.reset_interval) if self._options.has( + "reset_interval") else SignalBaseCacheManager.DEFAULT_RESET_INTERVAL if self.__reset_interval < 0: raise ValueError("Invalid input for reset_interval!") - signaler_options = self._options.signaler if self._options.has("signaler") else None - self._reset_signaler = SignalerFactory.create(self.reset, signaler_options) + signaler_options = self._options.signaler if self._options.has( + "signaler") else None + self._reset_signaler = SignalerFactory.create( + self.reset, signaler_options) if self.__reset_interval > 0 or self.__clean_interval > 0: loop = asyncio.get_event_loop() if self.__reset_interval > 0: loop.create_task(self.__reset_async(self.__reset_interval)) if self.__clean_interval > 0: loop.create_task(self.__clean_async(self.__clean_interval)) - - async def __reset_async(self, interval:"int"): + + async def __reset_async(self, interval: "int"): try: while True: await asyncio.sleep(interval) self.reset() - except: ... + except: + ... - async def __clean_async(self, interval:"int"): + async def __clean_async(self, interval: "int"): try: while True: await asyncio.sleep(interval) self.clean() - except: ... + except: + ... diff --git a/bclib/cache/signaler/factory.py b/bclib/cache/signaler/factory.py index abe5c31..0a43e1e 100644 --- a/bclib/cache/signaler/factory.py +++ b/bclib/cache/signaler/factory.py @@ -1,13 +1,14 @@ from abc import ABC -from ..signaler.base_signaler import BaseSignaler -from bclib.utility import DictEx -from ..signaler.no_signaler import NoSignaler -from ..signaler.rabbit_signaler import RabbitSignaller from typing import Callable +from bclib.cache.signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.no_signaler import NoSignaler +from bclib.cache.signaler.rabbit_signaler import RabbitSignaller +from bclib.utility import DictEx + class SignalerFactory(ABC): @staticmethod - def create(reset_cache_callback:"Callable", signaler_options:"DictEx"=None) -> "BaseSignaler": + def create(reset_cache_callback: "Callable", signaler_options: "DictEx" = None) -> "BaseSignaler": """ This is a factory function for create signaler @@ -26,5 +27,6 @@ def create(reset_cache_callback:"Callable", signaler_options:"DictEx"=None) -> " if signaler_type == "rabbit": return RabbitSignaller(reset_cache_callback, signaler_options) else: - raise ValueError(f"Unknown type for signaler ('${signaler_type}')") - return NoSignaler() \ No newline at end of file + raise ValueError( + f"Unknown type for signaler ('${signaler_type}')") + return NoSignaler() diff --git a/bclib/cache/signaler/no_signaler.py b/bclib/cache/signaler/no_signaler.py index 549e9cf..72a78d7 100644 --- a/bclib/cache/signaler/no_signaler.py +++ b/bclib/cache/signaler/no_signaler.py @@ -1,7 +1,8 @@ -from ..signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.base_signaler import BaseSignaler + class NoSignaler(BaseSignaler): """Implement no signaller for cache""" def __init__(self) -> None: - super().__init__(None, None) \ No newline at end of file + super().__init__(None, None) diff --git a/bclib/cache/signaler/rabbit_signaler.py b/bclib/cache/signaler/rabbit_signaler.py index cd23d25..12bf2b6 100644 --- a/bclib/cache/signaler/rabbit_signaler.py +++ b/bclib/cache/signaler/rabbit_signaler.py @@ -1,12 +1,14 @@ from typing import Callable import json import asyncio -from ..signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.base_signaler import BaseSignaler from bclib.utility import DictEx + class RabbitSignaller(BaseSignaler): """Implement rabbit-mq signaler""" - def __init__(self, reset_cache_callback:"Callable", options:"DictEx") -> None: + + def __init__(self, reset_cache_callback: "Callable", options: "DictEx") -> None: super().__init__(reset_cache_callback, options) import pika try: diff --git a/bclib/cache/value_item/array_value_item.py b/bclib/cache/value_item/array_value_item.py index 77591c5..a0a5cee 100644 --- a/bclib/cache/value_item/array_value_item.py +++ b/bclib/cache/value_item/array_value_item.py @@ -1,6 +1,6 @@ from bclib.cache.cache_item.base_cache_item import BaseCacheItem -from ..value_item.base_value_item import BaseValueItem -from ..cache_item.function_cache_item import FunctionCacheItem +from bclib.cache.value_item.base_value_item import BaseValueItem + class ArrayValueItem(BaseValueItem): diff --git a/bclib/cache/value_item/base_value_item.py b/bclib/cache/value_item/base_value_item.py index f236026..e7d8368 100644 --- a/bclib/cache/value_item/base_value_item.py +++ b/bclib/cache/value_item/base_value_item.py @@ -1,18 +1,19 @@ from abc import ABC, abstractmethod -from ..cache_item.base_cache_item import BaseCacheItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class BaseValueItem(ABC): - def __init__(self, cache_item:"BaseCacheItem") -> None: + def __init__(self, cache_item: "BaseCacheItem") -> None: super().__init__() - self._item:"list[BaseCacheItem]|BaseCacheItem" = None + self._item: "list[BaseCacheItem]|BaseCacheItem" = None self._apply_item(cache_item) - + @abstractmethod - def _apply_item(self, cache_item:"BaseCacheItem"): ... + def _apply_item(self, cache_item: "BaseCacheItem"): ... def add_or_update_item(self, cache_item: "BaseCacheItem"): self._apply_item(cache_item) - + def get_item(self) -> "list|any|None": if self._item is not None: if isinstance(self._item, list): @@ -27,7 +28,6 @@ def get_item(self) -> "list|any|None": else: ret_val = None return ret_val - + def reset(self): self._item = None - diff --git a/bclib/cache/value_item/scalar_value_item.py b/bclib/cache/value_item/scalar_value_item.py index b3dd327..58c50f1 100644 --- a/bclib/cache/value_item/scalar_value_item.py +++ b/bclib/cache/value_item/scalar_value_item.py @@ -1,7 +1,8 @@ from bclib.cache.cache_item.base_cache_item import BaseCacheItem -from ..value_item.base_value_item import BaseValueItem +from bclib.cache.value_item.base_value_item import BaseValueItem + class ScalarValueItem(BaseValueItem): - + def _apply_item(self, cache_item: "BaseCacheItem") -> "BaseCacheItem": self._item = cache_item diff --git a/bclib/context/__init__.py b/bclib/context/__init__.py index b9a8483..e69de29 100644 --- a/bclib/context/__init__.py +++ b/bclib/context/__init__.py @@ -1,14 +0,0 @@ -from bclib.context.client_source_context import ClientSourceContext -from bclib.context.client_source_member_context import ClientSourceMemberContext -from bclib.context.context import Context -from bclib.context.restful_context import RESTfulContext -from bclib.context.web_context import WebContext -from bclib.context.request_context import RequestContext -from bclib.context.rabbit_context import RabbitContext -from bclib.context.socket_context import SocketContext -from bclib.context.merge_type import MergeType -from bclib.context.server_source_context import ServerSourceContext -from bclib.context.server_source_member_context import ServerSourceMemberContext -from bclib.context.source_context import SourceContext -from bclib.context.source_member_context import SourceMemberContext -from bclib.context.named_pipe_context import NamedPipeContext diff --git a/bclib/context/client_source_context.py b/bclib/context/client_source_context.py index 647f127..2f74200 100644 --- a/bclib/context/client_source_context.py +++ b/bclib/context/client_source_context.py @@ -5,15 +5,15 @@ from bclib.context.json_base_request_context import JsonBaseRequestContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher import IDispatcher + from bclib.listener import WebMessage class ClientSourceContext(JsonBaseRequestContext): """Context for client dbSource request""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) parser = HtmlParserEx() self.raw_command = self.cms.form.command self.dmn_id = self.cms.form.dmnid if self.cms.form.has( diff --git a/bclib/context/client_source_member_context.py b/bclib/context/client_source_member_context.py index 5faea5a..40d94dc 100644 --- a/bclib/context/client_source_member_context.py +++ b/bclib/context/client_source_member_context.py @@ -1,22 +1,23 @@ -from abc import ABC from typing import Any -from .merge_type import MergeType -from .context import Context -from .client_source_context import ClientSourceContext +from bclib.context.merge_type import MergeType +from bclib.context.context import Context +from bclib.context.client_source_context import ClientSourceContext + +print(__name__) class ClientSourceMemberContext(Context): """Context for dbSource member request""" - def __init__(self, sourceContext: ClientSourceContext, data: Any, member: dict) -> None: - super().__init__(sourceContext.dispatcher) - self.__source_context = sourceContext + def __init__(self, source_context: 'ClientSourceContext', data: 'Any', member: 'dict') -> None: + super().__init__(source_context.dispatcher) + self.__source_context = source_context self.member = member self.data = data self.cms = self.__source_context.cms self.command = self.__source_context.command self.url = self.__source_context.url - self.table_name: str = f"{sourceContext.command.name}.{member.name}" + self.table_name: str = f"{source_context.command.name}.{member.name}" self.key_field_name: str = None self.status_field_name: str = None self.merge_type: MergeType = MergeType.REPLACE diff --git a/bclib/context/context.py b/bclib/context/context.py index e47923e..186a0cb 100644 --- a/bclib/context/context.py +++ b/bclib/context/context.py @@ -1,42 +1,27 @@ from abc import ABC import traceback -from typing import TYPE_CHECKING, Tuple +import base64 +from typing import TYPE_CHECKING, Optional, Tuple from bclib.exception import ShortCircuitErr - -from bclib.db_manager import SqlDb, SQLiteDb, MongoDb, RabbitConnection, RESTfulConnection from bclib.utility import DictEx, HttpStatusCodes, HttpStatusCodes from bclib.listener.http_listener import HttpBaseDataName, HttpBaseDataType -import base64 if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher import IDispatcher class Context(ABC): """Base class for dispatching""" - def __init__(self, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, dispatcher: 'IDispatcher') -> 'None': super().__init__() self.dispatcher = dispatcher - self.url_segments: DictEx = None - self.url: str = None + self.url_segments: 'DictEx' = None + self.url: 'Optional[str]' = None self.is_adhoc = True - def open_sql_connection(self, key: str) -> SqlDb: - return self.dispatcher.db_manager.open_sql_connection(key) - - def open_sqllite_connection(self, key: str) -> SQLiteDb: - return self.dispatcher.db_manager.open_sqllite_connection(key) - - def open_mongo_connection(self, key: str) -> MongoDb: - return self.dispatcher.db_manager.open_mongo_connection(key) - - def open_restful_connection(self, key: str) -> RESTfulConnection: - return self.dispatcher.db_manager.open_restful_connection(key) - - def open_rabbit_connection(self, key: str) -> RabbitConnection: - return self.dispatcher.db_manager.open_rabbit_connection(key) + # TODO:Removed wrapper open_X_connection methode def generate_error_response(self, exception: Exception) -> dict: """Generate error response from process result""" @@ -60,7 +45,8 @@ def _generate_error_object(self, exception: Exception) -> 'Tuple[dict, str]': "errorCode": error_code, "errorMessage": str(exception) } - if self.dispatcher.log_error: + # TODO:use logger service + if True: # self.dispatcher.log_error: error_object["error"] = traceback.format_exc() return (error_object, status_code) @@ -82,8 +68,9 @@ def _generate_response_cms( ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.INDEX] = response_type ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.HEADER_CODE] = status_code ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] = mime - if isinstance(content,bytes): - ret_val[HttpBaseDataType.CMS][HttpBaseDataName.BLOB_CONTENT] = base64.b64encode(content).decode("utf-8") + if isinstance(content, bytes): + ret_val[HttpBaseDataType.CMS][HttpBaseDataName.BLOB_CONTENT] = base64.b64encode( + content).decode("utf-8") else: ret_val[HttpBaseDataType.CMS][HttpBaseDataName.CONTENT] = content if headers is not None: diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py new file mode 100644 index 0000000..27b422d --- /dev/null +++ b/bclib/context/context_factory.py @@ -0,0 +1,132 @@ +import re +from struct import error +from typing import Callable, Optional, TYPE_CHECKING + + +if TYPE_CHECKING: + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.logger import ILogger +from bclib.utility import DictEx +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.request_context import RequestContext +from bclib.context.socket_context import SocketContext +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.end_point_context import EndPointContext +from bclib.listener.message import Message, MessageType +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + + +class ContextFactory: + def __init__(self, options: 'DictEx', logger: 'ILogger'): + self.logger = logger + self.__default_router = options.defaultRouter\ + if 'defaultRouter' in options and isinstance(options.defaultRouter, str)\ + else None + if options.has('router'): + router = options.router + if isinstance(router, str): + self.__context_type_detector: 'Callable[[str],str]' = lambda _: router + elif isinstance(router, DictEx): + self.__init_router_lookup(options.router) + else: + raise error( + "Invalid value for 'router' property in host options! Use string or dict object only.") + elif self.__default_router: + self.__context_type_detector: 'Callable[[str],str]' = lambda _: self.__default_router + else: + raise error( + "Invalid routing config! Please at least set one of 'router' or 'defaultRouter' property in host options.") + + def __init_router_lookup(self, router: 'DictEx'): + """create router lookup dictionary""" + + route_dict = dict() + for key, values in router.items(): + key = key.strip() + if key != 'rabbit': + if '*' in values: + route_dict['*'] = key + break + else: + for value in values: + if len(value.strip()) != 0 and value not in route_dict: + route_dict[value] = key + if len(route_dict) == 1 and '*' in route_dict and self.__default_router is None: + router = route_dict['*'] + self.__context_type_detector: 'Callable[[str],str]' = lambda _: router + else: + self.__context_type_lookup = route_dict.items() + self.__context_type_detector = self.__context_type_detect_from_lookup + + def __context_type_detect_from_lookup(self, url: str) -> str: + """Detect context type from url about lookup""" + + context_type: Optional[str] = None + if url: + try: + for pattern, lookup_context_type in self.__context_type_lookup: + if pattern == "*" or re.search(pattern, url): + context_type = lookup_context_type + break + except TypeError: + pass + except error as ex: + print("Error in detect context from routing options!", ex) + return context_type if context_type else self.__default_router + + async def create_context_async(self, message: 'Message', dispatcher: 'IDispatcher') -> Context: + """Create context from message object""" + + message_json: dict = await message.get_json_async() + + ret_val: RequestContext = None + context_type = None + cms_object: Optional[dict] = None + url: Optional[str] = None + request_id: Optional[str] = None + method: Optional[str] = None + + cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None + if cms_object: + if 'request' in cms_object: + req = cms_object["request"] + else: + raise KeyError("request key not found in cms object") + if 'full-url' in req: + url = req["full-url"] + else: + raise KeyError("full-url key not found in request") + request_id = req['request-id'] if 'request-id' in req else 'none' + method = req['methode'] if 'methode' in req else 'none' + if message.type == MessageType.AD_HOC: + if url or self.__default_router is None: + context_type = self.__context_type_detector(url) + else: + context_type = self.__default_router + else: + context_type = "endpoint" + self.logger.log_request( + f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") + + if context_type == "client_source": + ret_val = ClientSourceContext(cms_object, dispatcher, message) + elif context_type == "restful": + ret_val = RESTfulContext(cms_object, dispatcher, message) + elif context_type == "server_source": + ret_val = ServerSourceContext(message_json, dispatcher) + elif context_type == "web": + ret_val = WebContext(cms_object, dispatcher, message) + elif context_type == "endpoint": + ret_val = EndPointContext(cms_object, dispatcher, message, message_json) + elif context_type == "socket": + ret_val = SocketContext( + cms_object, dispatcher, message, message_json) + elif context_type is None: + raise NameError(f"No context found for '{url}'") + else: + raise NameError( + f"Configured context type '{context_type}' not found for '{url}'") + return ret_val \ No newline at end of file diff --git a/bclib/context/end_point_context.py b/bclib/context/end_point_context.py new file mode 100644 index 0000000..8f31b36 --- /dev/null +++ b/bclib/context/end_point_context.py @@ -0,0 +1,47 @@ +from typing import Any +from bclib.dispatcher.idispatcher import IDispatcher +from bclib.listener.end_point_message import EndPointMessage +from bclib.context.context import Context +from bclib.listener.message_type import MessageType +from bclib.utility import DictEx, HttpMimeTypes, HttpStatusCodes, ResponseTypes + + +import json + + +class EndPointContext(Context): + """Base class for dispatching end point socket base request context""" + + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher', message_object: 'EndPointMessage', body: 'dict') -> None: + super().__init__(dispatcher) + self.cms = DictEx(cms_object) if cms_object else None + self.url: str = self.cms.request.url if cms_object else None + self.body = DictEx(body) if body else None + self.message = message_object + self.is_adhoc = False + + async def send_object_async(self, obj) -> None: + await self.send_content_async(json.dumps(obj)) + + async def send_html_async(self, html) -> None: + await self.send_content_async(html, mime=HttpMimeTypes.HTML) + + async def send_code_async(self, code) -> None: + await self.send_content_async(code, response_type=ResponseTypes.RENDERABLE, mime=HttpMimeTypes.HTML) + + async def send_content_async(self, + content: 'Any', + response_type: 'str' = ResponseTypes.RENDERED, + status_code: 'str' = HttpStatusCodes.OK, + mime: 'str' = HttpMimeTypes.JSON, + template: 'DictEx' = None, + headers: 'dict' = None) -> None: + cms = Context._generate_response_cms( + content, response_type, status_code, mime, template, headers) + await self.message.write_result_async(cms,MessageType.MESSAGE) + + async def read_message_async(self) -> 'EndPointMessage': + return await self.message.read_next_message_async() + + async def send_close_async(self) -> None: + await self.message.write_result_async(None,MessageType.DISCONNECT) \ No newline at end of file diff --git a/bclib/context/json_base_request_context.py b/bclib/context/json_base_request_context.py index 0b4f0a6..0407cb3 100644 --- a/bclib/context/json_base_request_context.py +++ b/bclib/context/json_base_request_context.py @@ -5,24 +5,24 @@ from bclib.context.web_context import WebContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.web_message import WebMessage class JsonBaseRequestContext(WebContext): """Base class for dispatching http json base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) self.mime = HttpMimeTypes.JSON - def generate_error_response(self, exception: Exception) -> dict: + def generate_error_response(self, exception: 'Exception') -> 'dict': """Generate error response from process result""" error_object, self.status_code = self._generate_error_object(exception) self.mime = HttpMimeTypes.JSON return self.generate_response(error_object) - def generate_response(self, content: dict) -> dict: + def generate_response(self, content: 'dict') -> 'dict': """Generate response from process content""" return super().generate_response(json.dumps(content)) diff --git a/bclib/context/named_pipe_context.py b/bclib/context/named_pipe_context.py deleted file mode 100644 index c45a9aa..0000000 --- a/bclib/context/named_pipe_context.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import TYPE_CHECKING -from . import Context -if TYPE_CHECKING: - from .. import dispatcher -from bclib.utility import DictEx - - -class NamedPipeContext(Context): - """Context for named pipe request""" - - def __init__(self, named_pipe_message: 'dict', raw_message: str, dispatcher: 'dispatcher.IDispatcher'): - super().__init__(dispatcher) - self.message: DictEx = DictEx(named_pipe_message) - self.raw_message = raw_message diff --git a/bclib/context/rabbit_context.py b/bclib/context/rabbit_context.py index 9f4049e..fd4993d 100644 --- a/bclib/context/rabbit_context.py +++ b/bclib/context/rabbit_context.py @@ -1,15 +1,16 @@ import json -from typing import Any, TYPE_CHECKING -from ..context.context import Context +from typing import TYPE_CHECKING +from bclib.context.context import Context +from bclib.utility.dict_ex import DictEx + if TYPE_CHECKING: - from .. import dispatcher -from bclib.utility import DictEx + from bclib.dispatcher.idispatcher import IDispatcher class RabbitContext(Context): """Context for rabbit-mq request""" - def __init__(self, rabbit_message: DictEx, dispatcher: 'dispatcher.IDispatcher'): + def __init__(self, rabbit_message: 'DictEx', dispatcher: 'IDispatcher'): super().__init__(dispatcher) self.__rabbit_message: DictEx = rabbit_message self.__message = DictEx(json.loads( diff --git a/bclib/context/request_context.py b/bclib/context/request_context.py index d84e0c5..cbb4cf6 100644 --- a/bclib/context/request_context.py +++ b/bclib/context/request_context.py @@ -1,24 +1,24 @@ import json -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Optional from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes from bclib.exception import ShortCircuitErr from bclib.context.context import Context if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher.idispatcher import IDispatcher class RequestContext(Context): """Base class for dispatching http base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher') -> None: super().__init__(dispatcher) self.cms = DictEx(cms_object) self.url: str = self.cms.request.url self.query: DictEx = self.cms.query self.form: DictEx = self.cms.form - self.__headers: dict = None + self.__headers: Optional[dict] = None self.response_type: str = ResponseTypes.RENDERED self.status_code: str = HttpStatusCodes.OK self.mime = HttpMimeTypes.HTML @@ -40,7 +40,8 @@ def generate_error_response(self, exception: Exception) -> dict: content = exception.data if isinstance( exception.data, str) else json.dumps(exception.data, indent=1).replace("\n", "
") else: - content = f"{error_object['errorMessage']} (Error Code: {error_object['errorCode']})" + content = f"{error_object['errorMessage']} (Error Code: " +\ + f"{error_object['errorCode']})" if 'error' in error_object: error = error_object["error"].replace("\n", "
") content += f"
{error}" diff --git a/bclib/context/restful_context.py b/bclib/context/restful_context.py index eb3cdf8..68ce0ad 100644 --- a/bclib/context/restful_context.py +++ b/bclib/context/restful_context.py @@ -1,17 +1,17 @@ import json from typing import TYPE_CHECKING -from bclib.utility import DictEx -from ..context.json_base_request_context import JsonBaseRequestContext +from bclib.utility.dict_ex import DictEx +from bclib.context.json_base_request_context import JsonBaseRequestContext from urllib.parse import parse_qsl if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.web_message import WebMessage class RESTfulContext(JsonBaseRequestContext): - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) temp_data = None if self.cms.form: temp_data = self.cms.form diff --git a/bclib/context/server_source_context.py b/bclib/context/server_source_context.py index 4cb49f2..17b4b02 100644 --- a/bclib/context/server_source_context.py +++ b/bclib/context/server_source_context.py @@ -1,17 +1,17 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher.idispatcher import IDispatcher -from bclib.parser import HtmlParserEx -from bclib.utility import DictEx -from ..context.context import Context +from bclib.parser.html.html_parser_ex import HtmlParserEx +from bclib.utility.dict_ex import DictEx +from bclib.context.context import Context class ServerSourceContext(Context): """Base class for dispatching server base dbsource request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher') -> None: super().__init__(dispatcher) parser = HtmlParserEx() self.raw_command = cms_object["command"] diff --git a/bclib/context/server_source_member_context.py b/bclib/context/server_source_member_context.py index 0ecc264..f864ec4 100644 --- a/bclib/context/server_source_member_context.py +++ b/bclib/context/server_source_member_context.py @@ -1,20 +1,21 @@ -from typing import Any -from ..context.merge_type import MergeType -from ..context.context import Context -from ..context.server_source_context import ServerSourceContext +from typing import Any, TYPE_CHECKING, Optional +from bclib.context.merge_type import MergeType +from bclib.context.context import Context +if TYPE_CHECKING: + from bclib.context.server_source_context import ServerSourceContext class ServerSourceMemberContext(Context): """Context for Server dbSource member request""" - def __init__(self, sourceContext: ServerSourceContext, data: Any, member: dict) -> None: - super().__init__(sourceContext.dispatcher) - self.__source_context = sourceContext + def __init__(self, source_context: 'ServerSourceContext', data: Any, member: 'dict') -> None: + super().__init__(source_context.dispatcher) + self.__source_context = source_context self.member = member self.data = data self.command = self.__source_context.command - self.table_name: str = f"{sourceContext.command.name}.{member.name}" - self.key_field_name: str = None - self.status_field_name: str = None + self.table_name: str = f"{source_context.command.name}.{member.name}" + self.key_field_name: Optional[str] = None + self.status_field_name: Optional[str] = None self.merge_type: MergeType = MergeType.REPLACE self.column_names: 'list[str]' = None diff --git a/bclib/context/socket_context.py b/bclib/context/socket_context.py index 024e3f6..bbc5395 100644 --- a/bclib/context/socket_context.py +++ b/bclib/context/socket_context.py @@ -1,18 +1,19 @@ import json from typing import TYPE_CHECKING -from .context import Context -from bclib.listener.receive_message import ReceiveMessage + +from bclib.listener.message_type import MessageType +from bclib.context.context import Context from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, HttpStatusCodes, ResponseTypes if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.socket_message import SocketMessage class SocketContext(Context): """Base class for dispatching web socket base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher', message_object: 'listener.ReceiveMessage', body: dict) -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'SocketMessage', body: dict) -> None: super().__init__(dispatcher) self.cms = DictEx(cms_object) if cms_object else None self.url: str = self.cms.request.url if cms_object else None @@ -38,13 +39,10 @@ async def send_content_async(self, headers: 'dict' = None) -> None: cms = Context._generate_response_cms( content, response_type, status_code, mime, template, headers) - message = ReceiveMessage.create_from_object( - self.message.session_id, cms) - await message.write_to_stream_async(self.message.writer) + await self.message.write_result_async(cms, MessageType.MESSAGE) - async def read_message_async(self) -> 'listener.ReceiveMessage': + async def read_message_async(self) -> 'SocketMessage': return await self.message.read_next_message_async() async def send_close_async(self) -> None: - message = ReceiveMessage.create_disconnect(self.message.session_id) - await message.write_to_stream_async(self.message.writer) + await self.message.write_result_async(None, MessageType.DISCONNECT) diff --git a/bclib/context/source_context.py b/bclib/context/source_context.py index cde6933..4adb15d 100644 --- a/bclib/context/source_context.py +++ b/bclib/context/source_context.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx class SourceContext(ABC): diff --git a/bclib/context/source_member_context.py b/bclib/context/source_member_context.py index 89503bb..c300358 100644 --- a/bclib/context/source_member_context.py +++ b/bclib/context/source_member_context.py @@ -1,9 +1,9 @@ from abc import ABC, abstractmethod from typing import Any -from ..context.merge_type import MergeType +from bclib.context.merge_type import MergeType -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx class SourceMemberContext(ABC): diff --git a/bclib/context/web_context.py b/bclib/context/web_context.py index 1793f78..01b1eb6 100644 --- a/bclib/context/web_context.py +++ b/bclib/context/web_context.py @@ -3,51 +3,51 @@ from typing import Any, TYPE_CHECKING, Coroutine, Iterator, Optional, Union from aiohttp.web_response import ContentCoding -from ..context.request_context import RequestContext +from bclib.context.request_context import RequestContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.listener import WebMessage + from bclib.dispatcher.idispatcher import IDispatcher class WebContext(RequestContext): - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: super().__init__(cms_object, dispatcher) self.process_async = True self.__message = message_object - - async def start_stream_response_async(self,status: int = 200, - reason: Optional[str] = 'OK', - headers: Optional[dict] = None,)-> Coroutine[Any, Any, None]: - await self.__message.start_stream_response_async(status,reason,headers) - async def write_async(self,data:'bytes') -> Coroutine[Any, Any, None]: - await self.__message.write_async(data) + async def start_stream_response_async(self, status: int = 200, + reason: Optional[str] = 'OK', + headers: Optional[dict] = None,) -> Coroutine[Any, Any, None]: + await self.__message.start_stream_response_async(status, reason, headers) + + async def write_async(self, data: 'bytes') -> Coroutine[Any, Any, None]: + await self.__message.write_async(data) async def drain_async(self) -> Coroutine[Any, Any, None]: await self.__message.drain_async() - - async def write_and_drain_async(self,data:'bytes') -> Coroutine[Any, Any, None]: + + async def write_and_drain_async(self, data: 'bytes') -> Coroutine[Any, Any, None]: await self.write_async(data) await self.drain_async() - async def enable_compression(self,force: Optional[Union[bool, ContentCoding]] = None) -> None: + async def enable_compression(self, force: Optional[Union[bool, ContentCoding]] = None) -> None: await self.__message.enable_compression(force) - async def drain_array_async(self,data_list:Iterator,source_name:str, chunk_size:int,delimiter:str=',')-> Coroutine[Any, Any, None]: - total_len =len(data_list) - current:int = 0 + async def drain_array_async(self, data_list: Iterator, source_name: str, chunk_size: int, delimiter: str = ',') -> Coroutine[Any, Any, None]: + total_len = len(data_list) + current: int = 0 while current < total_len: - temp_list = list(islice(data_list,current,current + chunk_size)) - current+= len(temp_list) + temp_list = list(islice(data_list, current, current + chunk_size)) + current += len(temp_list) data = { "sources": [ - { - "options": { - "tableName": source_name, - "mergeType": 1 #MergeType append, - }, - "data": temp_list - }], + { + "options": { + "tableName": source_name, + "mergeType": 1 # MergeType append, + }, + "data": temp_list + }], } await self.write_and_drain_async(f"{json.dumps(data)}{delimiter}".encode()) diff --git a/bclib/db_manager/__init__.py b/bclib/db_manager/__init__.py index ff7fd85..de012b3 100644 --- a/bclib/db_manager/__init__.py +++ b/bclib/db_manager/__init__.py @@ -4,6 +4,3 @@ from bclib.db_manager.mongo_db import MongoDb from bclib.db_manager.rabbit_connection import RabbitConnection from bclib.db_manager.restful_connection import RESTfulConnection -from bclib.db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from bclib.db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection -from bclib.db_manager.named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection diff --git a/bclib/db_manager/db_manager.py b/bclib/db_manager/db_manager.py index a5b498f..a670c9f 100644 --- a/bclib/db_manager/db_manager.py +++ b/bclib/db_manager/db_manager.py @@ -1,13 +1,11 @@ import asyncio from bclib.utility import DictEx -from ..db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from ..db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection -from ..db_manager.rabbit_connection import RabbitConnection -from ..db_manager.db import Db -from ..db_manager.mongo_db import MongoDb -from ..db_manager.sql_db import SqlDb -from ..db_manager.sqlite_db import SQLiteDb -from ..db_manager.restful_connection import RESTfulConnection +from bclib.db_manager.rabbit_connection import RabbitConnection +from bclib.db_manager.db import Db +from bclib.db_manager.mongo_db import MongoDb +from bclib.db_manager.sql_db import SqlDb +from bclib.db_manager.sqlite_db import SQLiteDb +from bclib.db_manager.restful_connection import RESTfulConnection class DbManager: @@ -42,9 +40,6 @@ def open_connection(self, key: str) -> Db: ret_val = RESTfulConnection(setting) elif db_type == "rabbit": ret_val = RabbitConnection(setting) - elif db_type == "named_pipe": - ret_val = NamedPipeConnection.get_connection( - setting, self._event_loop) else: print( f"Data base of type '{db_type}' not supported in this version") @@ -64,6 +59,3 @@ def open_restful_connection(self, key: str) -> RESTfulConnection: def open_rabbit_connection(self, key: str) -> RabbitConnection: return self.open_connection(key) - - def get_named_pipe_connection(self, key: str) -> INamedPipeConnection: - return self.open_connection(key) diff --git a/bclib/db_manager/mongo_db.py b/bclib/db_manager/mongo_db.py index eeebc08..590d237 100644 --- a/bclib/db_manager/mongo_db.py +++ b/bclib/db_manager/mongo_db.py @@ -1,6 +1,3 @@ -from ..db_manager.db import Db - - class SingletonMeta(type): """ The Singleton class can be implemented in different ways in Python. Some diff --git a/bclib/db_manager/named_pipe/__init__.py b/bclib/db_manager/named_pipe/__init__.py deleted file mode 100644 index 5e9e312..0000000 --- a/bclib/db_manager/named_pipe/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bclib.db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from bclib.db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection -from bclib.db_manager.named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection diff --git a/bclib/db_manager/named_pipe/inamed_pipe_connection.py b/bclib/db_manager/named_pipe/inamed_pipe_connection.py deleted file mode 100644 index bf516fb..0000000 --- a/bclib/db_manager/named_pipe/inamed_pipe_connection.py +++ /dev/null @@ -1,13 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class INamedPipeConnection(ABC): - - @abstractmethod - def try_send_command(self, command: 'Any') -> bool: - pass - - @abstractmethod - async def try_send_query_async(self, query: 'Any') -> 'Any': - pass diff --git a/bclib/db_manager/named_pipe/linux_named_pipe_connection.py b/bclib/db_manager/named_pipe/linux_named_pipe_connection.py deleted file mode 100644 index 57311a8..0000000 --- a/bclib/db_manager/named_pipe/linux_named_pipe_connection.py +++ /dev/null @@ -1,78 +0,0 @@ -import asyncio -from io import BufferedReader, BufferedWriter -import json -import uuid -from typing import Any -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - -from bclib.utility import LinuxNamedPipeHelper - - -class LinuxNamedPipeConnection(INamedPipeConnection): - def __init__(self, pipe_name: str, event_loop: asyncio.AbstractEventLoop) -> None: - self.__event_loop = event_loop - self.__reader_pipe_name = F"{pipe_name}-reader" - self.__writer_pipe_name = F"{pipe_name}-writer" - self.__reader_pipe_handle: BufferedWriter = None - self.__writer_pipe_handle: BufferedReader = None - self.__query_list: 'dict[str,asyncio.Future]' = dict() - self.__propcess_tesk = self.__event_loop.create_task( - self.__get_receive_message_Task()) - - async def __get_receive_message_Task(self): - while True: - try: - if self.__writer_pipe_handle is None: - self.__writer_pipe_handle = open( - self.__writer_pipe_name, 'rb') - message = await LinuxNamedPipeHelper.read_from_named_pipe_async(self.__writer_pipe_handle, self.__event_loop) - if message and message.session_id in self.__query_list: - future = self.__query_list[message.session_id] - del self.__query_list[message.session_id] - data = json.loads(message.buffer) - future.get_loop().call_soon_threadsafe(future.set_result, data) - except asyncio.CancelledError: - self.clear_query_list() - break - except: - self.__writer_pipe_handle = None - self.clear_query_list() - await asyncio.sleep(1) - - def __try_send_command(self, command: Any) -> str: - from bclib.listener import Message - session_id = None - try: - if self.__reader_pipe_handle is None: - self.__reader_pipe_handle = open(self.__reader_pipe_name, "wb") - session_id = str(uuid.uuid4()) - msg = Message.create_add_hock( - session_id, json.dumps(command).encode("utf-8")) - LinuxNamedPipeHelper.write_to_named_pipe( - msg, self.__reader_pipe_handle) - except: - self.__reader_pipe_handle = None - raise - return session_id - - def try_send_command(self, command: Any) -> bool: - return True if self.__try_send_command(command) else False - - async def try_send_query_async(self, query: 'Any') -> 'Any': - ret_val = None - session_id = self.__try_send_command(query) - if session_id: - future = self.__event_loop.create_future() - self.__query_list[session_id] = future - ret_val = await future - return ret_val - - def clear_query_list(self): - for session_id in self.__query_list: - try: - future = self.__query_list[session_id] - future.get_loop().call_soon_threadsafe( - future.set_exception, Exception("Named Pipe is broken")) - except: - pass - self.__query_list.clear() diff --git a/bclib/db_manager/named_pipe/named_pipe_connection.py b/bclib/db_manager/named_pipe/named_pipe_connection.py deleted file mode 100644 index 9e32d8b..0000000 --- a/bclib/db_manager/named_pipe/named_pipe_connection.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -from sys import platform -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - - -class NamedPipeConnection: - __dic: 'dict[str,INamedPipeConnection]' = dict() - - @staticmethod - def get_connection(pipe_name: str, loop: asyncio.AbstractEventLoop) -> INamedPipeConnection: - ret_val = None - if pipe_name in NamedPipeConnection.__dic: - ret_val = NamedPipeConnection.__dic[pipe_name] - else: - # https://docs.python.org/3/library/sys.html#sys.platform - if platform == "linux" or platform == "linux2": - # linux - from ..named_pipe.linux_named_pipe_connection import LinuxNamedPipeConnection - ret_val = NamedPipeConnection.__dic[pipe_name] = LinuxNamedPipeConnection( - pipe_name, loop) - elif platform == "darwin": - # OS X - raise Exception( - "named pipe connection not implemented in OS X") - elif platform == "win32": - from ..named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection - ret_val = NamedPipeConnection.__dic[pipe_name] = WindowsNamedPipeConnection( - pipe_name, loop) - NamedPipeConnection.__dic[pipe_name] = ret_val - return ret_val diff --git a/bclib/db_manager/named_pipe/windows_named_pipe_connection.py b/bclib/db_manager/named_pipe/windows_named_pipe_connection.py deleted file mode 100644 index d54d439..0000000 --- a/bclib/db_manager/named_pipe/windows_named_pipe_connection.py +++ /dev/null @@ -1,104 +0,0 @@ -import asyncio -import json -import uuid -from typing import Any -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - -from bclib.utility import WindowsNamedPipeHelper - - -class WindowsNamedPipeConnection(INamedPipeConnection): - def __init__(self, pipe_name: str, event_loop: asyncio.AbstractEventLoop) -> None: - self.__event_loop = event_loop - self.__reader_pipe_name = F"{pipe_name}/reader" - self.__writer_pipe_name = F"{pipe_name}/writer" - self.__reader_pipe_handle = None - self.__writer_pipe_handle = None - self.__query_list: 'dict[str,asyncio.Future]' = dict() - self.__propcess_tesk = self.__event_loop.create_task( - self.__get_receive_message_Task()) - - def __connect_to_reader_named_pipe(self): - import win32file - import pywintypes - try: - self.__reader_pipe_handle = win32file.CreateFile(self.__reader_pipe_name, win32file.GENERIC_WRITE, - 0, None, win32file.OPEN_EXISTING, win32file.FILE_ATTRIBUTE_NORMAL, None) - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe '{self.__reader_pipe_name}'. {repr(e)}") - else: - print( - f"Named Pipe '{self.__reader_pipe_name}' error code {e.args[0]}. {repr(e)}") - raise - - def __connect_to_writer_pipe(self): - import win32file - import pywintypes - try: - self.__writer_pipe_handle = win32file.CreateFile(self.__writer_pipe_name, win32file.GENERIC_READ, - 0, None, win32file.OPEN_EXISTING, win32file.FILE_ATTRIBUTE_READONLY, None) - - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe '{self.__writer_pipe_name}'. {repr(e)}") - else: - print( - f"Named Pipe '{self.__writer_pipe_name}' error code {e.args[0]}. {repr(e)}") - raise - - async def __get_receive_message_Task(self): - while True: - try: - if self.__writer_pipe_handle is None: - self.__connect_to_writer_pipe() - message = await WindowsNamedPipeHelper.read_from_named_pipe_async(self.__writer_pipe_handle, self.__event_loop) - if message and message.session_id in self.__query_list: - future = self.__query_list[message.session_id] - del self.__query_list[message.session_id] - data = json.loads(message.buffer) - future.get_loop().call_soon_threadsafe(future.set_result, data) - except asyncio.CancelledError: - self.clear_query_list() - break - except: - self.__writer_pipe_handle = None - self.clear_query_list() - await asyncio.sleep(1) - - def __try_send_command(self, command: Any) -> str: - from bclib.listener import Message - session_id = None - try: - if self.__reader_pipe_handle is None: - self.__connect_to_reader_named_pipe() - session_id = str(uuid.uuid4()) - msg = Message.create_add_hock( - session_id, json.dumps(command).encode("utf-8")) - WindowsNamedPipeHelper.write_to_named_pipe( - msg, self.__reader_pipe_handle) - except: - self.__reader_pipe_handle = None - return session_id - - def try_send_command(self, command: Any) -> bool: - return True if self.__try_send_command(command) else False - - async def try_send_query_async(self, query: 'Any') -> 'Any': - ret_val = None - session_id = self.__try_send_command(query) - if session_id: - future = self.__event_loop.create_future() - self.__query_list[session_id] = future - ret_val = await future - return ret_val - - def clear_query_list(self): - for session_id in self.__query_list: - try: - future = self.__query_list[session_id] - future.get_loop().call_soon_threadsafe( - future.set_exception, Exception("Named Pipe is broken")) - except: - pass - self.__query_list.clear() diff --git a/bclib/db_manager/odbc_db.py b/bclib/db_manager/odbc_db.py index f0ecbe5..afcaf40 100644 --- a/bclib/db_manager/odbc_db.py +++ b/bclib/db_manager/odbc_db.py @@ -2,7 +2,7 @@ Implementation of ODBC base Db object https://github.com/mkleehammer/pyodbc/wiki """ -from ..db_manager.db import Db +from bclib.db_manager.db import Db class OdbcDb(Db): diff --git a/bclib/db_manager/rabbit_connection.py b/bclib/db_manager/rabbit_connection.py index f1e8b99..a9bc8e1 100644 --- a/bclib/db_manager/rabbit_connection.py +++ b/bclib/db_manager/rabbit_connection.py @@ -2,7 +2,7 @@ import json from typing import Any from bclib.utility import DictEx -from ..db_manager.db import Db +from bclib.db_manager.db import Db class RabbitConnection(Db): diff --git a/bclib/db_manager/restful_connection.py b/bclib/db_manager/restful_connection.py index 48d4bc4..a818780 100644 --- a/bclib/db_manager/restful_connection.py +++ b/bclib/db_manager/restful_connection.py @@ -1,6 +1,6 @@ """RESTful implementation of Db wrapper""" from typing import Any -from ..db_manager.db import Db +from bclib.db_manager.db import Db class RESTfulConnection(Db): diff --git a/bclib/db_manager/sql_db.py b/bclib/db_manager/sql_db.py index 8877fa7..6399ccb 100644 --- a/bclib/db_manager/sql_db.py +++ b/bclib/db_manager/sql_db.py @@ -1,4 +1,4 @@ -from ..db_manager.odbc_db import OdbcDb +from bclib.db_manager.odbc_db import OdbcDb class SqlDb(OdbcDb): diff --git a/bclib/db_manager/sqlite_db.py b/bclib/db_manager/sqlite_db.py index 6fc48f3..df9ffa6 100644 --- a/bclib/db_manager/sqlite_db.py +++ b/bclib/db_manager/sqlite_db.py @@ -1,5 +1,5 @@ import sqlite3 -from ..db_manager.db import Db +from bclib.db_manager.db import Db class SQLiteDb(Db): diff --git a/bclib/dispatcher/__init__.py b/bclib/dispatcher/__init__.py index a149245..e69de29 100644 --- a/bclib/dispatcher/__init__.py +++ b/bclib/dispatcher/__init__.py @@ -1,7 +0,0 @@ -from bclib.dispatcher.dispatcher import Dispatcher -from bclib.dispatcher.idispatcher import IDispatcher -from bclib.dispatcher.socket_dispatcher import SocketDispatcher -from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher -from bclib.dispatcher.routing_dispatcher import RoutingDispatcher -from bclib.dispatcher.named_pipe_dispatcher import NamedPipeDispatcher -from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher diff --git a/bclib/dispatcher/callback_info.py b/bclib/dispatcher/callback_info.py index 1ea3df0..0804d2b 100644 --- a/bclib/dispatcher/callback_info.py +++ b/bclib/dispatcher/callback_info.py @@ -1,15 +1,18 @@ -from typing import Any, Callable, Awaitable -from ..context import Context -from ..predicate import Predicate +from typing import TYPE_CHECKING from bclib.exception import ShortCircuitErr +if TYPE_CHECKING: + from typing import Any, Callable, Awaitable + from bclib.predicate import Predicate + from bclib.context.context import Context + class CallbackInfo: - def __init__(self, predicates: 'list[Predicate]', async_callback: 'Callable[[Context], Awaitable[Any]]') -> Any: + def __init__(self, predicates: 'list[Predicate]', async_callback: 'Callable[[Context], Awaitable[Any]]') -> 'Any': self.__async_callback = async_callback self.__predicates = predicates - async def try_execute_async(self, context: Context) -> Any: + async def try_execute_async(self, context: 'Context') -> 'Any': result: Any = None for predicate in self.__predicates: try: diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index 5023d1d..82560fb 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -1,17 +1,27 @@ import asyncio -from ..dispatcher.socket_dispatcher import RoutingDispatcher -from bclib.listener import Endpoint, HttpListener + +from dependency_injector import containers + +from bclib.context.context_factory import ContextFactory +from bclib.logger.ilogger import ILogger +from bclib.cache.cache_manager import CacheManager +from bclib.db_manager import DbManager +from bclib.utility import DictEx +from bclib.listener.endpoint import Endpoint +from bclib.listener.http_listener import HttpListener +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class DevServerDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__listener = HttpListener( Endpoint(self.options.server), self._on_message_receive_async, self.options.ssl, self.options.configuration - ) + ) def initialize_task(self): super().initialize_task() diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index e7dd7c6..b9a67e8 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -3,59 +3,86 @@ import inspect from abc import ABC import signal -import sys -import traceback -from typing import Callable, Any, Coroutine, Optional +from typing import Awaitable, Callable, Any, Coroutine, Optional, TYPE_CHECKING from functools import wraps -from bclib.logger import ILogger, LoggerFactory, LogObject -from bclib.cache import CacheFactory -from bclib.listener import RabbitBusListener +from dependency_injector import containers +from listener.rabbit_bus_listener import RabbitBusListener + +from bclib.logger.ilogger import ILogger +from cache.cache_manager import CacheManager from bclib.predicate import Predicate -from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext, NamedPipeContext -from bclib.db_manager import DbManager +from bclib.db_manager.db_manager import DbManager from bclib.utility import DictEx from bclib.exception import HandlerNotFoundErr -from ..dispatcher.callback_info import CallbackInfo +from bclib.dispatcher.callback_info import CallbackInfo + +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.client_source_member_context import ClientSourceMemberContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.rabbit_context import RabbitContext +from bclib.context.socket_context import SocketContext +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.server_source_member_context import ServerSourceMemberContext +from bclib.context.end_point_context import EndPointContext + +if TYPE_CHECKING: + from bclib.logger.ilogger import LogObject class Dispatcher(ABC): """Base class for dispatching request""" - def __init__(self, options: dict = None,loop:asyncio.AbstractEventLoop = None): - self.options = DictEx(options) + def __init__(self, container: 'containers.Container', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop'): + self.options = options + self.container = container self.__look_up: 'dict[str, list[CallbackInfo]]' = dict() - cache_options = self.options.cache if "cache" in self.options else None - if loop is None and sys.platform == 'win32': - # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. - # Use Windows version of proactor event loop using IOCP - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - self.event_loop = asyncio.get_event_loop() if loop is None else loop - self.cache_manager = CacheFactory.create(cache_options) - self.db_manager = DbManager(self.options, self.event_loop) - self.__logger: ILogger = LoggerFactory.create(self.options) - self.log_error: bool = self.options.log_error if self.options.has( - "log_error") else False - self.log_request: bool = self.options.log_request if self.options.has( - "log_request") else True + self.event_loop: 'asyncio.AbstractEventLoop' = loop + self.cache_manager: 'CacheManager' = cache_manager + self.db_manager = db_manager + self.logger: ILogger = logger self.__rabbit_dispatcher: 'list[RabbitBusListener]' = list() if "router" in self.options and "rabbit" in self.options.router: for setting in self.options.router.rabbit: self.__rabbit_dispatcher.append( RabbitBusListener(setting, self)) + def endpoint_action(self, * predicates: (Predicate)): + """Decorator for determine end point action""" + + def _decorator(end_point_action_handler: 'Callable[[EndPointContext],bool]'): + + @wraps(end_point_action_handler) + async def non_async_wrapper(context: 'EndPointContext'): + await self.event_loop.run_in_executor(None, end_point_action_handler, context) + return True + + @wraps(end_point_action_handler) + async def async_wrapper(context: 'EndPointContext'): + await end_point_action_handler(context) + return True + + wrapper = async_wrapper if inspect.iscoroutinefunction( + end_point_action_handler) else non_async_wrapper + + self._get_context_lookup(EndPointContext.__name__)\ + .append(CallbackInfo([*predicates], wrapper)) + return end_point_action_handler + return _decorator + def socket_action(self, * predicates: (Predicate)): """Decorator for determine Socket action""" def _decorator(socket_action_handler: 'Callable[[SocketContext],bool]'): @wraps(socket_action_handler) - async def non_async_wrapper(context: SocketContext): + async def non_async_wrapper(context: 'SocketContext'): await self.event_loop.run_in_executor(None, socket_action_handler, context) return True @wraps(socket_action_handler) - async def async_wrapper(context: SocketContext): + async def async_wrapper(context: 'SocketContext'): await socket_action_handler(context) return True @@ -73,12 +100,12 @@ def restful_action(self, * predicates: (Predicate)): def _decorator(restful_action_handler: 'Callable[[RESTfulContext], dict]'): @wraps(restful_action_handler) - async def non_async_wrapper(context: RESTfulContext): + async def non_async_wrapper(context: 'RESTfulContext'): action_result = await self.event_loop.run_in_executor(None, restful_action_handler, context) return None if action_result is None else context.generate_response(action_result) @wraps(restful_action_handler) - async def async_wrapper(context: RESTfulContext): + async def async_wrapper(context: 'RESTfulContext'): action_result = await restful_action_handler(context) return None if action_result is None else context.generate_response(action_result) @@ -96,12 +123,12 @@ def web_action(self, * predicates: (Predicate)): def _decorator(web_action_handler: 'Callable[[WebContext], str]'): @wraps(web_action_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): action_result = await self.event_loop.run_in_executor(None, web_action_handler, context) return None if action_result is None else context.generate_response(action_result) @wraps(web_action_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): action_result = await web_action_handler(context) return None if action_result is None else context.generate_response(action_result) @@ -118,7 +145,7 @@ def client_source_action(self, *predicates: (Predicate)): def _decorator(client_source_action_handler: 'Callable[[ClientSourceContext], Any]'): @wraps(client_source_action_handler) - async def non_async_wrapper(context: ClientSourceContext): + async def non_async_wrapper(context: 'ClientSourceContext'): data = await self.event_loop.run_in_executor(None, client_source_action_handler, context) result_set = list() if data is not None: @@ -148,7 +175,7 @@ async def non_async_wrapper(context: ClientSourceContext): return None @wraps(client_source_action_handler) - async def async_wrapper(context: ClientSourceContext): + async def async_wrapper(context: 'ClientSourceContext'): data = await client_source_action_handler(context) result_set = list() if data is not None: @@ -192,11 +219,11 @@ def client_source_member_action(self, *predicates: (Predicate)): def _decorator(client_source_member_handler: 'Callable[[ClientSourceMemberContext], Any]'): @wraps(client_source_member_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): return await self.event_loop.run_in_executor(None, client_source_member_handler, context) @wraps(client_source_member_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): return await client_source_member_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -212,7 +239,7 @@ def server_source_action(self, *predicates: (Predicate)): def _decorator(server_source_action_handler: 'Callable[[ServerSourceContext], Any]'): @wraps(server_source_action_handler) - async def non_async_wrapper(context: ServerSourceContext): + async def non_async_wrapper(context: 'ServerSourceContext'): data = await self.event_loop.run_in_executor(None, server_source_action_handler, context) result_set = list() if data is not None: @@ -242,7 +269,7 @@ async def non_async_wrapper(context: ServerSourceContext): return None @wraps(server_source_action_handler) - async def async_wrapper(context: ServerSourceContext): + async def async_wrapper(context: 'ServerSourceContext'): data = await server_source_action_handler(context) result_set = list() if data is not None: @@ -286,11 +313,11 @@ def server_source_member_action(self, *predicates: (Predicate)): def _decorator(server_source_member_action_handler: 'Callable[[ServerSourceMemberContext], Any]'): @wraps(server_source_member_action_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): return await self.event_loop.run_in_executor(None, server_source_member_action_handler, context) @wraps(server_source_member_action_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): return await server_source_member_action_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -307,11 +334,11 @@ def rabbit_action(self, * predicates: (Predicate)): def _decorator(rabbit_action_handler: 'Callable[[RabbitContext], bool]'): @wraps(rabbit_action_handler) - async def non_async_wrapper(context: RabbitContext): + async def non_async_wrapper(context: 'RabbitContext'): return await self.event_loop.run_in_executor(None, rabbit_action_handler, context) @wraps(rabbit_action_handler) - async def async_wrapper(context: RabbitContext): + async def async_wrapper(context: 'RabbitContext'): return await rabbit_action_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -323,28 +350,6 @@ async def async_wrapper(context: RabbitContext): return rabbit_action_handler return _decorator - def named_pipe_action(self, * predicates: (Predicate)): - """Decorator for determine named pipe message request action""" - - def _decorator(named_pipe_action_handler: 'Callable[[RabbitContext], bool]'): - - @wraps(named_pipe_action_handler) - async def non_async_wrapper(context: NamedPipeContext): - return await self.event_loop.run_in_executor(None, named_pipe_action_handler, context) - - @wraps(named_pipe_action_handler) - async def async_wrapper(context: NamedPipeContext): - return await named_pipe_action_handler(context) - - wrapper = async_wrapper if inspect.iscoroutinefunction( - named_pipe_action_handler) else non_async_wrapper - - self._get_context_lookup(NamedPipeContext.__name__)\ - .append(CallbackInfo([*predicates], wrapper)) - - return named_pipe_action_handler - return _decorator - def _get_context_lookup(self, key: str) -> 'list[CallbackInfo]': """Get key related action list object""" @@ -356,7 +361,7 @@ def _get_context_lookup(self, key: str) -> 'list[CallbackInfo]': self.__look_up[key] = ret_val return ret_val - async def dispatch_async(self, context: Context) -> Any: + async def dispatch_async(self, context: 'Context') -> Any: """Dispatch context and get result from related action method""" result: Any = None @@ -369,12 +374,10 @@ async def dispatch_async(self, context: Context) -> Any: break else: ex = HandlerNotFoundErr(name) - if self.log_error: - print(str(ex)) + self.logger.log_error(ex) result = context.generate_error_response(ex) except Exception as ex: - if self.log_error: - traceback.print_exc() + self.logger.log_error(ex) result = context.generate_error_response(ex) return result @@ -387,12 +390,14 @@ def initialize_task(self): for dispatcher in self.__rabbit_dispatcher: dispatcher.initialize_task(self.event_loop) - def listening(self, before_start: Coroutine=None, after_end: Coroutine=None,with_block:bool = True): + # TODO:pre ansd post callback replaced with resource provider of DI + def listening(self, with_block: bool = True): """Start listening to request for process""" for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda sig, _: self.event_loop.stop()) - if before_start != None: - self.event_loop.run_until_complete(self.event_loop.create_task(before_start())) + init_process = self.container.init_resources() + if isinstance(init_process, Awaitable): + self.event_loop.run_until_complete(init_process) self.initialize_task() if with_block: self.event_loop.run_forever() @@ -401,23 +406,24 @@ def listening(self, before_start: Coroutine=None, after_end: Coroutine=None,with task.cancel() group = asyncio.gather(*tasks, return_exceptions=True) self.event_loop.run_until_complete(group) - if after_end != None: - self.event_loop.run_until_complete(self.event_loop.create_task(after_end())) + shutdown_process = self.container.shutdown_resources() + if isinstance(shutdown_process, Awaitable): + self.event_loop.run_until_complete(shutdown_process) self.event_loop.close() - def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: - return self.__logger.new_object_log(schema_name, routing_key, **kwargs) + def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> 'LogObject': + return self.logger.new_object_log(schema_name, routing_key, **kwargs) - async def log_async(self, log_object: LogObject = None, **kwargs): + async def log_async(self, log_object: 'LogObject' = None, **kwargs): """log params""" if log_object is None: if "schema_name" not in kwargs: raise Exception("'schema_name' not set for apply logging!") schema_name = kwargs.pop("schema_name") log_object = self.new_object_log(schema_name, **kwargs) - await self.__logger.log_async(log_object) + await self.logger.log_async(log_object) - def log_in_background(self, log_object: LogObject = None, **kwargs) -> Coroutine: + def log_in_background(self, log_object: 'LogObject' = None, **kwargs) -> Coroutine: """log params in background precess""" return self.event_loop.create_task( self.log_async(log_object, **kwargs) diff --git a/bclib/dispatcher/dispatcher_helper.py b/bclib/dispatcher/dispatcher_helper.py index b72565d..a980c5b 100644 --- a/bclib/dispatcher/dispatcher_helper.py +++ b/bclib/dispatcher/dispatcher_helper.py @@ -3,7 +3,7 @@ from bclib.predicate import Predicate, InList, Equal, Url, Between, NotEqual, GreaterThan, LessThan, LessThanEqual, GreaterThanEqual, Match, HasValue, Callback, All from bclib import predicate -from bclib.context import Context +from bclib.context.context import Context class DispatcherHelper: diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index da734ec..2223184 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -1,12 +1,20 @@ import asyncio -from ..listener import Endpoint, ReceiveMessage -from .routing_dispatcher import RoutingDispatcher +from dependency_injector import containers +from bclib.listener.end_point_message import EndPointMessage +from bclib.context.context_factory import ContextFactory +from cache.cache_manager import CacheManager +from bclib.db_manager import DbManager +from bclib.logger.ilogger import ILogger +from bclib.utility import DictEx +from bclib.listener.endpoint import Endpoint +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class EndpointDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__endpoint = Endpoint(self.options.endpoint) def initialize_task(self): @@ -14,9 +22,8 @@ def initialize_task(self): async def on_connection_open(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): try: - msg = await ReceiveMessage.read_from_stream_async(reader, writer) - result = await self._on_message_receive_async(msg) - await result.write_to_stream_async(writer) + msg = EndPointMessage(reader, writer) + await self._on_message_receive_async(message=msg) except: pass try: diff --git a/bclib/dispatcher/idispatcher.py b/bclib/dispatcher/idispatcher.py index 2ba3a7f..10be69f 100644 --- a/bclib/dispatcher/idispatcher.py +++ b/bclib/dispatcher/idispatcher.py @@ -1,20 +1,22 @@ """Dispatcher base class module""" from abc import ABC, abstractmethod import asyncio +from dependency_injector import containers from typing import Callable, Any, TYPE_CHECKING, Coroutine, Optional -from bclib.db_manager import DbManager -from bclib.cache import CacheManager -from bclib.listener import Message +from bclib.db_manager.db_manager import DbManager +from cache.cache_manager import CacheManager from bclib.utility import DictEx -from bclib.logger import LogObject +from bclib.logger.log_object import LogObject if TYPE_CHECKING: - from context import Context + from bclib.context.context import Context class IDispatcher(ABC): """Dispatcher base class with core functionality for manage cache and background process""" + container: 'containers.Container' + @property @abstractmethod def log_error(self) -> bool: @@ -53,10 +55,6 @@ async def dispatch_async(self, context: 'Context') -> Any: def dispatch_in_background(self, context: 'Context') -> asyncio.Future: """Dispatch context in background""" - @abstractmethod - async def send_message_async(self, message: Message) -> None: - """Send message to endpoint""" - def run_in_background(self, callback: Callable, *args: Any) -> asyncio.Future: """helper for run function in background thread""" diff --git a/bclib/dispatcher/named_pipe_dispatcher.py b/bclib/dispatcher/named_pipe_dispatcher.py deleted file mode 100644 index a53e6af..0000000 --- a/bclib/dispatcher/named_pipe_dispatcher.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio -from sys import platform -from ..dispatcher.routing_dispatcher import RoutingDispatcher -from bclib.listener import Message - - -class NamedPipeDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) - self.__lock = asyncio.Lock() - # https://docs.python.org/3/library/sys.html#sys.platform - if platform == "linux" or platform == "linux2": - # linux - from bclib.listener import LinuxNamedPipeListener - self.__listener = LinuxNamedPipeListener( - self.options.named_pipe, - self._on_message_receive_async) - elif platform == "win32": - from bclib.listener import WindowsNamedPipeListener - self.__listener = WindowsNamedPipeListener( - self.options.named_pipe, - self._on_message_receive_async) - elif platform == "darwin": - # OS X - raise Exception("named pipe not implemented in OS X") - - async def send_message_async(self, message: Message) -> bool: - """Send message to endpoint""" - - async with self.__lock: - return await self.__listener.send_message_async(message) - - def initialize_task(self): - super().initialize_task() - self.__listener.initialize_task(self.event_loop) diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index e48a9b5..8eadadb 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -1,150 +1,37 @@ import asyncio import inspect -import json -import re -from struct import error -from typing import Callable, Any, Coroutine, Optional -from bclib.utility import DictEx - -from bclib.context import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext, NamedPipeContext -from bclib.listener import Message, MessageType, HttpBaseDataType, ReceiveMessage +from typing import Callable, Any, Coroutine +from dependency_injector import containers +from bclib.logger.ilogger import ILogger +from bclib.db_manager.db_manager import DbManager +from cache.cache_manager import CacheManager +from bclib.utility import DictEx +from bclib.context.context_factory import ContextFactory +from bclib.listener.message import Message, MessageType from bclib.dispatcher.dispatcher_helper import DispatcherHelper from bclib.dispatcher.dispatcher import Dispatcher class RoutingDispatcher(Dispatcher, DispatcherHelper): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) - self.__default_router = self.options.defaultRouter\ - if 'defaultRouter' in self.options and isinstance(self.options.defaultRouter, str)\ - else None - self.name = self.options["name"] if self.options.has("name") else None - self.__log_name = f"{self.name}: " if self.name else '' - if self.options.has('router'): - router = self.options.router - if isinstance(router, str): - self.__context_type_detector: 'Callable[[str],str]' = lambda _: router - elif isinstance(router, DictEx): - self.init_router_lookup() - else: - raise error( - "Invalid value for 'router' property in host options! Use string or dict object only.") - elif self.__default_router: - self.__context_type_detector: 'Callable[[str],str]' = lambda _: self.__default_router - else: - raise error( - "Invalid routing config! Please at least set one of 'router' or 'defaultRouter' property in host options.") - - def init_router_lookup(self): - """create router lookup dictionary""" - - route_dict = dict() - for key, values in self.options.router.items(): - if key != 'rabbit'.strip(): - if '*' in values: - route_dict['*'] = key - break - else: - for value in values: - if len(value.strip()) != 0 and value not in route_dict: - route_dict[value] = key - if len(route_dict) == 1 and '*' in route_dict and self.__default_router is None: - router = route_dict['*'] - self.__context_type_detector: 'Callable[[str],str]' = lambda _: router - else: - self.__context_type_lookup = route_dict.items() - self.__context_type_detector = self.__context_type_detect_from_lookup - - def __context_type_detect_from_lookup(self, url: str) -> str: - """Detect context type from url about lookup""" - - context_type: str = None - if url: - try: - for pattern, lookup_context_type in self.__context_type_lookup: - if pattern == "*" or re.search(pattern, url): - context_type = lookup_context_type - break - except TypeError: - pass - except error as ex: - print("Error in detect context from routing options!", ex) - return context_type if context_type else self.__default_router + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) + self._context_factory = context_factory - async def _on_message_receive_async(self, message: Message) -> Message: + async def _on_message_receive_async(self, message: Message) -> Coroutine: """Process received message""" try: - context = self.__context_factory(message) + context = await self._context_factory.create_context_async(message, self) response = await self.dispatch_async(context) - ret_val: Message = None - if context.is_adhoc: - ret_val = message.create_response_message( - message.session_id, - json.dumps(response, ensure_ascii=False).encode("utf-8") - ) - return ret_val + await message.set_result_async(response) except Exception as ex: print(f"Error in process received message {ex}") raise ex - def __context_factory(self, message: Message) -> Context: - """Create context from message object""" - - ret_val: RequestContext = None - context_type = None - cms_object: Optional[dict] = None - url: Optional[str] = None - request_id: Optional[str] = None - method: Optional[str] = None - message_json: Optional[dict] = None - if message.buffer is not None: - message_json = json.loads(message.buffer) - cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None - if cms_object: - if 'request' in cms_object: - req = cms_object["request"] - else: - raise KeyError("request key not found in cms object") - if 'full-url' in req: - url = req["full-url"] - else: - raise KeyError("full-url key not found in request") - request_id = req['request-id'] if 'request-id' in req else 'none' - method = req['methode'] if 'methode' in req else 'none' - if message.type == MessageType.AD_HOC: - if url or self.__default_router is None: - context_type = self.__context_type_detector(url) - else: - context_type = self.__default_router - else: - context_type = "socket" - if self.log_request: - print( - f"{self.__log_name}({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") - - if context_type == "client_source": - ret_val = ClientSourceContext(cms_object, self,message) - elif context_type == "restful": - ret_val = RESTfulContext(cms_object, self,message) - elif context_type == "server_source": - ret_val = ServerSourceContext(message_json, self) - elif context_type == "web": - ret_val = WebContext(cms_object, self,message) - elif context_type == "socket": - ret_val = SocketContext(cms_object, self, message, message_json) - elif context_type == "named_pipe": - ret_val = NamedPipeContext(message_json, message.buffer.decode("utf-8"), self) - elif context_type is None: - raise NameError(f"No context found for '{url}'") - else: - raise NameError( - f"Configured context type '{context_type}' not found for '{url}'") - return ret_val - def run_in_background(self, callback: 'Callable|Coroutine', *args: Any) -> asyncio.Future: """helper for run function in background thread""" @@ -158,7 +45,7 @@ async def send_message_async(self, message: MessageType) -> bool: raise NotImplementedError( "Send ad-hoc message not support in this type of dispatcher") - def cache(self, life_time:"int"=0, key:"str"=None): + def cache(self, life_time: "int" = 0, key: "str" = None): """Cache result of function for seconds of time or until signal by key for clear""" return self.cache_manager.cache_decorator(key, life_time) diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index 7227ef5..d38987b 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -1,23 +1,25 @@ import asyncio -from ..dispatcher.routing_dispatcher import RoutingDispatcher -from ..listener import Endpoint, Message, SocketListener +from dependency_injector import containers + +from bclib.context.context_factory import ContextFactory +from cache.cache_manager import CacheManager +from bclib.db_manager import DbManager +from bclib.logger.ilogger import ILogger +from bclib.utility import DictEx +from bclib.listener.endpoint import Endpoint +from bclib.listener.socket_listener import SocketListener +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class SocketDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) - self.__lock = asyncio.Lock() + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__listener = SocketListener( Endpoint(self.options.receiver), Endpoint(self.options.sender), self._on_message_receive_async) - async def send_message_async(self, message: Message) -> bool: - """Send message to endpoint""" - - async with self.__lock: - return await self.__listener.send_message_async(message) - def initialize_task(self): super().initialize_task() self.__listener.initialize_task(self.event_loop) diff --git a/bclib/edge.py b/bclib/edge.py index 46285bd..6546d82 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -1,13 +1,46 @@ """Main module of bclib.wrapper for all exist module that need in basic coding""" import asyncio -from bclib.db_manager import * -from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, NamedPipeDispatcher, EndpointDispatcher -from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, NamedPipeContext -from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders, WindowsNamedPipeHelper -from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName +from dependency_injector import providers +# from bclib.db_manager import * +# from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher +# from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, EndPointContext +from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders +# from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate from bclib.exception import * +from bclib.edge_container import EdgeContainer + +from bclib.db_manager import DbManager, SqlDb, SQLiteDb, MongoDb, RabbitConnection, RESTfulConnection + +from bclib.listener.message import Message +from bclib.listener.message_type import MessageType +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + + +from bclib.dispatcher.idispatcher import IDispatcher +from bclib.dispatcher.socket_dispatcher import SocketDispatcher +from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher +from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher + +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.client_source_member_context import ClientSourceMemberContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.request_context import RequestContext +from bclib.context.rabbit_context import RabbitContext +from bclib.context.socket_context import SocketContext +from bclib.context.merge_type import MergeType +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.server_source_member_context import ServerSourceMemberContext +from bclib.context.source_context import SourceContext +from bclib.context.source_member_context import SourceMemberContext +from bclib.context.end_point_context import EndPointContext + + from bclib import __version__ @@ -30,69 +63,77 @@ def from_list(hosts: 'dict[str,list[str]]'): __print_splash(True) loop = asyncio.get_event_loop() with concurrent.futures.ThreadPoolExecutor(max_workers=len(hosts.items())) as executor: - tasks:list[asyncio.Future] = [] + tasks: list[asyncio.Future] = [] for host, args in hosts.items(): - args.append(f"-n {host}") - args.append("-m") - tasks.append(loop.run_in_executor(executor, subprocess.run, args)) - print(f'{host} start running from {args[1]}') - loop.run_until_complete(asyncio.gather(*tasks)) + args.append(f"-n {host}") + args.append("-m") + tasks.append(loop.run_in_executor(executor, subprocess.run, args)) + print(f'{host} start running from {args[1]}') + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except KeyboardInterrupt: + pass -def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: - """Create related RoutingDispatcher obj from config object""" +def from_options(options: 'dict', loop: 'asyncio.AbstractEventLoop' = None) -> 'RoutingDispatcher': + container = EdgeContainer() + container.app_config.from_dict(options) + if loop: + container.app_config.loop.from_value(loop) + return from_container(container) + +def __get_arg_parts(container: 'EdgeContainer'): import sys import getopt - multi: bool = False - argumentList = sys.argv[1:] + argument_list = sys.argv[1:] # Options short_options = "mn:" # Long options long_options = ["Name =", "Multi"] + is_multi = False try: arguments, _ = getopt.gnu_getopt( - argumentList, short_options, long_options) + argument_list, short_options, long_options) for current_argument, current_value in arguments: if current_argument in ("-n", "--Name"): - options["name"] = current_value.strip() + container.app_config.name.from_value(current_value.strip()) elif current_argument in ("-m", "--Multi"): - multi = True + is_multi = True + container.app_config.is_multi.from_value(is_multi) except getopt.error as err: print(str(err)) - if not multi: + +def from_container(container: 'EdgeContainer') -> 'RoutingDispatcher': + """Create related RoutingDispatcher obj from config object""" + + if type(container) is not EdgeContainer: + container.app_container.override(providers.Object(container)) + __get_arg_parts(container) + if not container.app_config.is_multi(): __print_splash(False) - ret_val: RoutingDispatcher = None - if "server" in options: - ret_val = DevServerDispatcher(options=options,loop=loop) - elif "named_pipe" in options: - ret_val = NamedPipeDispatcher(options=options,loop=loop) - elif "endpoint" in options: - ret_val = EndpointDispatcher(options=options,loop=loop) - else: - ret_val = SocketDispatcher(options=options,loop=loop) - return ret_val - - -def __print_splash(inMultiMode: bool): + return container.dispatcher() + + +def __print_splash(in_multi_mode: bool): print(f''' -______ _ _____ _ -| ___ \\ (_) | ___| | | -| |_/ / __ _ ___ _ ___ ___ ___ _ __ ___ | |__ __| | __ _ ___ +______ _ _____ _ +| ___ \\ (_) | ___| | | +| |_/ / __ _ ___ _ ___ ___ ___ _ __ ___ | |__ __| | __ _ ___ | ___ \\/ _` / __| / __|/ __/ _ \\| '__/ _ \\ | __|/ _` |/ _` |/ _ \\ | |_/ / (_| \\__ \\ \\__ \\ (_| (_) | | | __/ | |__| (_| | (_| | __/ \\____/ \\__,_|___/_|___/\\___\\___/|_| \\___| \\____/\\__,_|\\__, |\\___| - __/ | - |___/ + __/ | + |___/ *********************************** Basiscore Edge Welcome To BasisCore Ecosystem Follow us on https://BasisCore.com/ bclib Version : {__version__} -Run in {'multi' if inMultiMode else 'single'} instance mode! +Run in {'multi' if in_multi_mode else 'single'} instance mode! *********************************** (Press CTRL+C to quit) ''') diff --git a/bclib/edge_container.py b/bclib/edge_container.py new file mode 100644 index 0000000..cab6ea5 --- /dev/null +++ b/bclib/edge_container.py @@ -0,0 +1,63 @@ +import asyncio +import sys + +from dependency_injector import containers, providers +from bclib.logger.logger_factory import LoggerFactory +from bclib.db_manager import DbManager +from bclib.cache.factory import CacheFactory +from bclib.utility import DictEx +from bclib.dispatcher.socket_dispatcher import SocketDispatcher +from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher +from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher +from bclib.context.context_factory import ContextFactory + + +def get_mode(options: 'DictEx'): + if options.has("server"): + ret_val = 'server' + elif options.has("endpoint"): + ret_val = 'endpoint' + else: + ret_val = 'socket' + return ret_val + + +def create_loop(options: 'DictEx') -> asyncio.AbstractEventLoop: + loop: asyncio.AbstractEventLoop = options.loop + if loop is None and sys.platform == 'win32': + # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. + # Use Windows version of proactor event loop using IOCP + loop = asyncio.ProactorEventLoop() + current_loop = asyncio.get_event_loop() + if loop is not None and current_loop != loop: + asyncio.set_event_loop(loop) + return asyncio.get_event_loop() + + +class EdgeContainer(containers.DeclarativeContainer): + app_config = providers.Configuration() + app_container = providers.Object(providers.Self()) + app_options = providers.Singleton(DictEx, app_config) + app_mode = providers.Singleton(get_mode, app_options) + app_cache_options = providers.Singleton(lambda x: x.cache, app_options) + app_event_loop = providers.Singleton(create_loop, app_options) + app_cache_manager = providers.Singleton( + CacheFactory.create, app_cache_options) + app_db_manager = providers.Singleton( + DbManager, app_options, app_event_loop) + app_logger = providers.Singleton(LoggerFactory.create, app_options) + app_context_factory = providers.Singleton( + ContextFactory, app_options, app_logger) + app_server_dispatcher = providers.Singleton( + DevServerDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) + app_endpoint_dispatcher = providers.Singleton( + EndpointDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) + app_socket_dispatcher = providers.Singleton( + SocketDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) + + dispatcher = providers.Selector( + app_mode, + server=app_server_dispatcher, + endpoint=app_endpoint_dispatcher, + socket=app_socket_dispatcher + ) diff --git a/bclib/exception/__init__.py b/bclib/exception/__init__.py index 907ffb7..f345a8f 100644 --- a/bclib/exception/__init__.py +++ b/bclib/exception/__init__.py @@ -4,4 +4,4 @@ from bclib.exception.not_found_err import NotFoundErr from bclib.exception.handler_not_found_err import HandlerNotFoundErr from bclib.exception.bad_request_err import BadRequestErr -from bclib.exception.forbidden_err import ForbiddenErr \ No newline at end of file +from bclib.exception.forbidden_err import ForbiddenErr diff --git a/bclib/exception/bad_request_err.py b/bclib/exception/bad_request_err.py index 053b714..cb470df 100644 --- a/bclib/exception/bad_request_err.py +++ b/bclib/exception/bad_request_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from .short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class BadRequestErr(ShortCircuitErr): diff --git a/bclib/exception/forbidden_err.py b/bclib/exception/forbidden_err.py index b6705a2..cbe81c5 100644 --- a/bclib/exception/forbidden_err.py +++ b/bclib/exception/forbidden_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from .short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class ForbiddenErr(ShortCircuitErr): diff --git a/bclib/exception/handler_not_found_err.py b/bclib/exception/handler_not_found_err.py index 2bcad3c..03ff470 100644 --- a/bclib/exception/handler_not_found_err.py +++ b/bclib/exception/handler_not_found_err.py @@ -1,4 +1,4 @@ -from ..exception.not_found_err import NotFoundErr +from bclib.exception.not_found_err import NotFoundErr class HandlerNotFoundErr(NotFoundErr): diff --git a/bclib/exception/internal_server_err.py b/bclib/exception/internal_server_err.py index 2ed8d20..b3d97d8 100644 --- a/bclib/exception/internal_server_err.py +++ b/bclib/exception/internal_server_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class InternalServerErr(ShortCircuitErr): diff --git a/bclib/exception/not_found_err.py b/bclib/exception/not_found_err.py index 6363f98..e5580d5 100644 --- a/bclib/exception/not_found_err.py +++ b/bclib/exception/not_found_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class NotFoundErr(ShortCircuitErr): diff --git a/bclib/exception/unauthorized_err.py b/bclib/exception/unauthorized_err.py index 8a9e157..21cdff5 100644 --- a/bclib/exception/unauthorized_err.py +++ b/bclib/exception/unauthorized_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class UnauthorizedErr(ShortCircuitErr): diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index aa15891..e69de29 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -1,12 +0,0 @@ -from bclib.listener.endpoint import Endpoint -from bclib.listener.socket_listener import SocketListener -from bclib.listener.rabbit_bus_listener import RabbitBusListener -from bclib.listener.message import Message -from bclib.listener.receive_message import ReceiveMessage -from bclib.listener.message_type import MessageType -from bclib.listener.http_listener.http_listener import HttpListener -from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName -from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType -from bclib.listener.windows_named_pipe_listener import WindowsNamedPipeListener -from bclib.listener.linux_named_pipe_listener import LinuxNamedPipeListener -from bclib.listener.web_message import WebMessage diff --git a/bclib/listener/end_point_message.py b/bclib/listener/end_point_message.py new file mode 100644 index 0000000..9735c94 --- /dev/null +++ b/bclib/listener/end_point_message.py @@ -0,0 +1,12 @@ +import asyncio + +from bclib.listener.stream_base_message import StreamBaseMessage + +class EndPointMessage(StreamBaseMessage): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): + super().__init__(reader, writer) + + async def read_next_message_async(self) -> 'EndPointMessage': + ret_val = EndPointMessage(self.reader, self.writer) + await ret_val._fill_async() + return ret_val diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 1cd2581..a539f32 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -1,27 +1,14 @@ import asyncio -import cgi -import io -import datetime -import json -import uuid import ssl -import base64 -from typing import Callable, TYPE_CHECKING, Optional, Awaitable -from urllib.parse import unquote, parse_qs - -from bclib.listener.message_type import MessageType -from ..endpoint import Endpoint -from ..http_listener.http_base_data_name import HttpBaseDataName -from ..http_listener.http_base_data_type import HttpBaseDataType -from bclib.utility import DictEx, ResponseTypes -from ..message import Message -from ..web_message import WebMessage -import pathlib +from typing import Callable, TYPE_CHECKING, Coroutine, Optional +from aiohttp import web +from aiohttp.log import web_logger +from bclib.utility.dict_ex import DictEx +from bclib.listener.web_message import WebMessage if TYPE_CHECKING: - from aiohttp import web + from bclib.listener.endpoint import Endpoint -from aiohttp.log import web_logger class HttpListener: _id = 0 @@ -36,76 +23,33 @@ class HttpListener: _DEFAULT_MIDDLEWARES = () _DEFAULT_HANDLER_ARGS = None _DEFAULT_CLIENT_MAX_SIZE = 1024 ** 2 - - def __init__(self, endpoint: Endpoint, async_callback: 'Callable[[WebMessage], Awaitable[WebMessage]]',ssl_options:'dict', configuration: Optional[DictEx]): + def __init__(self, endpoint: 'Endpoint', async_callback: 'Callable[[WebMessage],Coroutine]', ssl_options: 'dict', configuration: 'Optional[DictEx]'): self.__endpoint = endpoint self.on_message_receive_async = async_callback self.ssl_options = ssl_options self.__config = configuration if configuration is not None else DictEx() - self.__logger = self.__config.get(HttpListener.LOGGER, HttpListener._DEFAULT_LOGGER) - self.__router = self.__config.get(HttpListener.ROUTER, HttpListener._DEFAULT_ROUTER) - self.__middlewares = self.__config.get(HttpListener.MIDDLEWARES, HttpListener._DEFAULT_MIDDLEWARES) - self.__handler_args = self.__config.get(HttpListener.HANDLER_ARGS, HttpListener._DEFAULT_HANDLER_ARGS) - self.__client_max_size = self.__config.get(HttpListener.CLIENT_MAX_SIZE, HttpListener._DEFAULT_CLIENT_MAX_SIZE) - + self.__logger = self.__config.get( + HttpListener.LOGGER, HttpListener._DEFAULT_LOGGER) + self.__router = self.__config.get( + HttpListener.ROUTER, HttpListener._DEFAULT_ROUTER) + self.__middlewares = self.__config.get( + HttpListener.MIDDLEWARES, HttpListener._DEFAULT_MIDDLEWARES) + self.__handler_args = self.__config.get( + HttpListener.HANDLER_ARGS, HttpListener._DEFAULT_HANDLER_ARGS) + self.__client_max_size = self.__config.get( + HttpListener.CLIENT_MAX_SIZE, HttpListener._DEFAULT_CLIENT_MAX_SIZE) def initialize_task(self, event_loop: asyncio.AbstractEventLoop): event_loop.create_task(self.__server_task(event_loop)) async def __server_task(self, event_loop: asyncio.AbstractEventLoop): from aiohttp import web - from multidict import MultiDict + async def on_request_receive_async(request: 'web.Request') -> web.Response: - ret_val: web.Response = None - request_cms = await self.create_cms_async(request) - msg = WebMessage(request, str(uuid.uuid4()),MessageType.AD_HOC,json.dumps(request_cms, ensure_ascii=False).encode(encoding="utf-8")) - result = await self.on_message_receive_async(msg) - if result and result.Response is None: - cms: dict = json.loads(result.buffer.decode("utf-8")) - cms_cms = cms[HttpBaseDataType.CMS] - cms_cms_webserver = cms_cms[HttpBaseDataType.WEB_SERVER] - index = cms_cms_webserver[HttpBaseDataName.INDEX] - header_code: str = cms_cms_webserver[HttpBaseDataName.HEADER_CODE] - mime = cms_cms[HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] - headers = MultiDict() - if HttpBaseDataName.HTTP in cms_cms: - http: dict = cms_cms[HttpBaseDataName.HTTP] - for key, value in http.items(): - if isinstance(value, list): - for item in value: - headers.add(key, item) - else: - headers.add(key, value) - headers.add("Content-Type", mime) - if index == ResponseTypes.STATIC_FILE: - try: - path = pathlib.Path(cms_cms_webserver[HttpBaseDataName.FILE_PATH]) - path.stat() - ret_val = web.FileResponse( - path=path, - chunk_size=256*1024, - status=int(header_code.split(' ')[0]), - headers=headers - ) - except FileNotFoundError: - ret_val = web.Response( - status=404, - reason="File not found" - ) - else: - ret_val = web.Response( - status=int(header_code.split(' ')[0]), - headers=headers - ) - if HttpBaseDataName.CONTENT in cms_cms: - ret_val.text = cms_cms[HttpBaseDataName.CONTENT] - else: - raw_blob_content = cms_cms[HttpBaseDataName.BLOB_CONTENT] - ret_val.body = base64.b64decode(raw_blob_content.encode("utf-8")) - else: - ret_val = web.Response() if result.Response is None else result.Response - return ret_val + msg = WebMessage(request) + await self.on_message_receive_async(msg) + return msg.Response app = web.Application( logger=self.__logger, @@ -117,7 +61,7 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: ) app.add_routes( [web.route('*', '/{tail:.*}', on_request_receive_async)]) - ssl_context= None + ssl_context = None if self.ssl_options: ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) if 'certfile' in self.ssl_options: @@ -133,11 +77,13 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: ) ssl_context.load_cert_chain(certfile=pem_file_path) except Exception as e: - raise Exception("Invalid PKCS12 or pastphrase for {0}: {1}".format(self.ssl_options.pfxfile, e)) + raise Exception("Invalid PKCS12 or pastphrase for {0}: {1}".format( + self.ssl_options.pfxfile, e)) runner = web.AppRunner(app, handle_signals=True) await runner.setup() - site = web.TCPSite(runner, self.__endpoint.url, self.__endpoint.port, ssl_context=ssl_context) + site = web.TCPSite(runner, self.__endpoint.url, + self.__endpoint.port, ssl_context=ssl_context) await site.start() print( f"Development Edge server started at http{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port}") @@ -147,140 +93,27 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: except asyncio.CancelledError: pass finally: - print(f"Development Edge server for http{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port} stopped.") + print(f"Development Edge server for http" + + f"{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port} stopped.") await site.stop() await runner.cleanup() await runner.shutdown() @staticmethod - def convert_pfx_to_pem_file(pfxfile:str,password:str)->str: + def convert_pfx_to_pem_file(pfxfile: str, password: str) -> str: from cryptography.hazmat.primitives.serialization import pkcs12, Encoding, PrivateFormat, NoEncryption - with open(pfxfile,"rb") as f: + with open(pfxfile, "rb") as f: try: - private_key, certificate, additional_certificates = pkcs12.load_key_and_certificates(f.read(), password.encode()) + private_key, certificate, additional_certificates = pkcs12.load_key_and_certificates( + f.read(), password.encode()) pem_file_path = '{0}.auto-generated.pem'.format(pfxfile) with open(pem_file_path, 'wb') as pem_file: pem_file.write(certificate.public_bytes(Encoding.PEM)) for item in additional_certificates: pem_file.write(item.public_bytes(Encoding.PEM)) - pem_file.write(private_key.private_bytes(encoding= Encoding.PEM,format=PrivateFormat.TraditionalOpenSSL,encryption_algorithm=NoEncryption())) + pem_file.write(private_key.private_bytes( + encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, encryption_algorithm=NoEncryption())) return pem_file_path except Exception as ex: - raise Exception("Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) - - @staticmethod - async def create_cms_async(request: 'web.Request') -> dict: - cms_object = dict() - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.METHODE, request.method.lower()) - raw_url = unquote(request.path_qs)[1:] - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.RAW_URL, raw_url) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.URL, request.path[1:]) - HttpListener.__add_query_string(request.query, cms_object) - for key, value in request.headers.items(): - field_name = key.strip().lower() - if field_name == HttpBaseDataName.COOKIE: - HttpListener.__add_cookie(value, cms_object) - elif field_name == HttpBaseDataName.HOST: - HttpListener.__add_host(value, raw_url, cms_object) - else: - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - field_name, str(value).strip()) - HttpListener.__add_server_data(cms_object, request) - await HttpListener.__add_body_async(cms_object, request) - return {"cms": cms_object} - - @staticmethod - async def __add_body_async(cms_object: dict, request: 'web.Request'): - content_len_str = request.headers.get('Content-Length') - if content_len_str or request.can_read_body: - raw_body = await request.read() - body = raw_body.decode('utf-8') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.BODY, body) - content_type: str = request.headers.get( - "content-type") - if content_type and content_type.find("application/json") < 0: - if content_type.find("multipart/form-data") >= 0: - _, content_type_value_params = cgi.parse_header( - content_type) - content_type_value_params['boundary'] = bytes( - content_type_value_params['boundary'], "utf-8") - if content_len_str is not None: - content_type_value_params['CONTENT-LENGTH'] = int( - content_len_str) - with io.BytesIO(raw_body) as stream: - fields = cgi.parse_multipart( - stream, content_type_value_params) - for key, value in fields.items(): - HttpListener.__add_header(cms_object, - HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) - else: - for key, value in parse_qs(body).items(): - HttpListener.__add_header( - cms_object, HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) - - @staticmethod - def __add_server_data(cms_object: dict, request: 'web.Request'): - HttpListener._id += 1 - now = datetime.datetime.now() - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE, now.strftime("%d/%m/%Y")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.TIME, now.strftime("%H:%M")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE2, now.strftime("%Y%m%d")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.TIME2, now.strftime("%H%M%S")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE3, now.strftime("%Y.%m.%d")) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.REQUEST_ID, str(HttpListener._id)) - host_parts = request.host.split(':') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST_IP, host_parts[0]) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST_PORT, host_parts[1] if len(host_parts) > 1 else "80") # edit - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.CLIENT_IP, str(request.remote)) - - @staticmethod - def __add_query_string(query: dict, cms_object: dict) -> None: - for key, value in query.items(): - HttpListener.__add_header( - cms_object, HttpBaseDataType.QUERY, key, value) - - @staticmethod - def __add_cookie(raw_header_value: str, cms_object) -> None: - for item in raw_header_value.split(';'): - parts = item.split('=') - if len(parts) == 2: - HttpListener.__add_header(cms_object, HttpBaseDataName.COOKIE, - parts[0].strip(), parts[1].strip()) - - @staticmethod - def __add_host(row_host: str, row_url: str, cms_object) -> None: - host_parts = row_host.split(':') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST, host_parts[0]) - if len(host_parts) == 2: - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.PORT, host_parts[1]) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.FULL_URL, f"{row_host}/{row_url}") - - @staticmethod - def __add_header(cms_object: dict, value_type: str, value_name: str, value: str) -> None: - if value_type not in cms_object: - cms_object[value_type] = dict() - type_node = cms_object[value_type] - if value_name in type_node: - name_node = type_node[value_name] - if isinstance(name_node, list): - name_node.append(value) - else: - type_node[value_name] = [name_node, value] - else: - type_node[value_name] = value + raise Exception( + "Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) diff --git a/bclib/listener/linux_named_pipe_listener.py b/bclib/listener/linux_named_pipe_listener.py deleted file mode 100644 index f824d14..0000000 --- a/bclib/listener/linux_named_pipe_listener.py +++ /dev/null @@ -1,113 +0,0 @@ -import asyncio -from io import BufferedWriter -import os -import sys -from typing import Callable, Coroutine -from ..listener.message import Message -from bclib.utility import LinuxNamedPipeHelper - - -class LinuxNamedPipeListener: - """""" - - def __init__(self, pipe_name: str, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): - self.pipe_name = pipe_name - self.__writer_pipe_name = F"{self.pipe_name}-writer" - self.on_message_receive = on_message_receive_call_back - self.__writer_pipe: BufferedWriter = None - self.__event_loop: asyncio.AbstractEventLoop = None - - def try_unlink(self, pipe_name: str): - try: - os.unlink(pipe_name) - print(f"unlink named pipe '{pipe_name}'...") - except: - pass - - async def __connect_writer_pipe_async(self): - if self.__writer_pipe: - self.try_unlink(self.__writer_pipe_name) - - try: - os.mkfifo(self.__writer_pipe_name) - except Exception as ex: - print('error in run os.mkfifo():', ex) - - try: - self.__writer_pipe = open(self.__writer_pipe_name, "wb") - print( - f"Writer named pipe '{self.__writer_pipe_name}' is created...") - except Exception as ex: - print(f"Error in create writer named pipe. {repr(ex)}") - self.try_unlink(self.__writer_pipe_name) - raise - - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - - async def send_message_async(self, message: Message) -> bool: - try_count = 0 - send = False - while not send: - try: - if self.__writer_pipe is None: - await self.__connect_writer_pipe_async() - LinuxNamedPipeHelper.write_to_named_pipe( - message, self.__writer_pipe) - send = True - except asyncio.CancelledError: - break - except Exception as ex: - try_count = try_count+1 - if self.__writer_pipe: - self.try_unlink(self.__writer_pipe_name) - try: - self.__writer_pipe.close() - except: - pass - self.__writer_pipe = None - print(f"Error in send message {ex}") - if try_count > 3: - break - await asyncio.sleep(.5) - return send - - def initialize_task(self, loop: asyncio.AbstractEventLoop): - self.__event_loop = loop - - async def reader_loop_async(): - reader_pipe_name = F"{self.pipe_name}-reader" - while True: - try: - os.mkfifo(reader_pipe_name) - except Exception as ex: - print( - f'Error in call os.mkfifo() for ${reader_pipe_name}', ex) - - with open(reader_pipe_name, "rb") as reader_pipe: - print( - f"Reader named pipe '{reader_pipe_name}' is created. try to read from it...") - try: - while True: - message = await LinuxNamedPipeHelper.read_from_named_pipe_async(reader_pipe, self.__event_loop) - if message: - self.__event_loop.create_task( - self.__process_message_async(message)) - except asyncio.CancelledError: - print('Edge named pipe server stopped.!') - break - except Exception as ex: - print('Error cause edge named pipe server restart!', ex) - await asyncio.sleep(1) - finally: - try: - if reader_pipe: - os.unlink(reader_pipe_name) - print("unlink...") - except Exception as ex: - print("Error in unlink named pipe...", ex) - - self.__event_loop.create_task(reader_loop_async()) - self.__event_loop.create_task(self.__connect_writer_pipe_async()) diff --git a/bclib/listener/message.py b/bclib/listener/message.py index 72ae779..8a9477b 100644 --- a/bclib/listener/message.py +++ b/bclib/listener/message.py @@ -1,69 +1,19 @@ -import asyncio -import json -from typing import Any -from bclib.listener.message_type import MessageType +from typing import Any, Coroutine from abc import abstractmethod +from bclib.listener.message_type import MessageType -class Message: - def __init__(self, sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - self.session_id = sessionId - self.type = messageType - self.buffer = buffer +class Message: + def __init__(self ) -> None: + self.session_id: str = "not-set" + self.type: MessageType = MessageType.AD_HOC + @abstractmethod - def create_response_message(self, session_id: str, buffer: bytes) -> "Message": - return Message.create_add_hock(session_id,buffer) - - - async def write_to_stream_async(self, stream: asyncio.StreamWriter) -> bool: - is_send = True - try: - stream.write(self.type.value.to_bytes(1, 'big')) - data = self.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - stream.write(data_length_bytes) - stream.write(data) + async def get_json_async(self)-> Coroutine[Any, Any, dict]: + pass - if self.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(self.buffer).to_bytes(4, 'big') - stream.write(data_length_bytes) - stream.write(self.buffer) - await stream.drain() - except asyncio.CancelledError: - is_send = False - return is_send - - @staticmethod - def create_add_hock(session_id: str, buffer: bytes): - return Message(session_id, MessageType.AD_HOC, buffer) - - @staticmethod - def create_disconnect(session_id: str): - return Message(session_id, MessageType.DISCONNECT, None) - - @staticmethod - def create_from_text(session_id: str, text: str): - return Message(session_id, MessageType.MESSAGE, text.encode()) - - @staticmethod - def create_from_byte(session_id: str, array: bytes): - return Message(session_id, MessageType.MESSAGE, array) - - @staticmethod - def create_from_object(session_id: str, object_data: Any): - return Message(session_id, MessageType.MESSAGE, json.dumps(object_data).encode("utf-8")) + @abstractmethod + async def set_result_async(self, result: dict)-> Coroutine[Any, Any, None]: + pass - @staticmethod - def create(session_id: str, data: Any): - ret_val: Message = None - if isinstance(data, str): - ret_val = Message.create_from_text( - session_id, data) - elif isinstance(data, bytes): - ret_val = Message.create_from_byte( - session_id, data) - else: - ret_val = Message.create_from_object( - session_id, data) - return ret_val diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index a3d8a61..6470063 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -1,16 +1,16 @@ from struct import error -from bclib.context import RabbitContext from typing import TYPE_CHECKING +from bclib.context.rabbit_context import RabbitContext from bclib.listener.rabbit_listener import RabbitListener -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher.idispatcher import IDispatcher class RabbitBusListener(RabbitListener): - def __init__(self, rabbit_options: DictEx, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, rabbit_options: 'DictEx', dispatcher: 'IDispatcher') -> None: super().__init__(rabbit_options) self.__dispatcher = dispatcher diff --git a/bclib/listener/rabbit_listener.py b/bclib/listener/rabbit_listener.py index 5b698d9..a8443da 100644 --- a/bclib/listener/rabbit_listener.py +++ b/bclib/listener/rabbit_listener.py @@ -1,7 +1,7 @@ import asyncio from abc import ABC, abstractmethod -from bclib.utility import DictEx import pika +from bclib.utility import DictEx class RabbitListener(ABC): def __init__(self, connection_options: DictEx) -> None: diff --git a/bclib/listener/receive_message.py b/bclib/listener/receive_message.py deleted file mode 100644 index 7a42c90..0000000 --- a/bclib/listener/receive_message.py +++ /dev/null @@ -1,36 +0,0 @@ -import asyncio - -from .message_type import MessageType -from .message import Message - - -class ReceiveMessage(Message): - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - super().__init__(sessionId, messageType, buffer) - self.reader = reader - self.writer = writer - - async def read_next_message_async(self) -> 'ReceiveMessage': - return await ReceiveMessage.read_from_stream_async(self.reader, self.writer) - - @staticmethod - async def read_from_stream_async(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> 'ReceiveMessage': - message: ReceiveMessage = None - data = await reader.readexactly(1) - if data: - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = await reader.readexactly(4) - data_len = int.from_bytes(data, byteorder='big', signed=True) - data = await reader.readexactly(data_len) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = await reader.readexactly(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = await reader.readexactly(data_len) - parameter = data - message = ReceiveMessage( - reader, writer, session_id, message_type, parameter) - return message diff --git a/bclib/listener/socket_listener.py b/bclib/listener/socket_listener.py index 42746bb..11103d0 100644 --- a/bclib/listener/socket_listener.py +++ b/bclib/listener/socket_listener.py @@ -1,30 +1,19 @@ import asyncio from typing import Callable, Coroutine -from ..listener.message import Message -from ..listener.endpoint import Endpoint + +from bclib.listener.socket_message import SocketMessage +from bclib.listener.endpoint import Endpoint class SocketListener: - def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): + def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call_back: 'Callable[[SocketMessage],Coroutine]'): self.__receiver_endpoint = receiver self.__sender_endpoint = sender self.on_message_receive = on_message_receive_call_back self.__sender_stream_writer: asyncio.StreamWriter = None self.__receiver_server: asyncio.AbstractServer = None self.__sender_server: asyncio.AbstractServer = None - - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - - async def send_message_async(self, message: Message) -> bool: - try: - await message.write_to_stream_async(self.__sender_stream_writer) - except Exception as ex: - print(f"Error in send message {ex}") - return False - + async def on_sender_client_connect(self, _: asyncio.StreamReader, writer: asyncio.StreamWriter): peer_name = writer.get_extra_info('peername') print(f'Reader from {peer_name}, connect to sender!') @@ -60,9 +49,9 @@ async def on_receiver_client_connect(self, reader: asyncio.StreamReader, writer: cause = "closed!" try: while True: - message = await Message.read_from_stream_async(reader) + message = SocketMessage(reader, writer) if message: - loop.create_task(self.__process_message_async(message)) + loop.create_task(self.on_message_receive(message)) except asyncio.CancelledError: cause = 'closed by receiver!' except (ConnectionResetError, asyncio.IncompleteReadError): diff --git a/bclib/listener/socket_message.py b/bclib/listener/socket_message.py new file mode 100644 index 0000000..f6fbd0a --- /dev/null +++ b/bclib/listener/socket_message.py @@ -0,0 +1,7 @@ +from bclib.listener.stream_base_message import StreamBaseMessage + +import asyncio + +class SocketMessage(StreamBaseMessage): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): + super().__init__(reader, writer) \ No newline at end of file diff --git a/bclib/listener/stream_base_message.py b/bclib/listener/stream_base_message.py new file mode 100644 index 0000000..a7cfaf8 --- /dev/null +++ b/bclib/listener/stream_base_message.py @@ -0,0 +1,54 @@ +import asyncio +import json +from typing import Any, Coroutine, Optional + +from bclib.listener.message_type import MessageType +from bclib.listener.message import Message + + +class StreamBaseMessage(Message): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): + self.reader = reader + self.writer = writer + self.buffer: Optional[bytes] = None + + async def get_json_async(self) -> Coroutine[Any, Any, dict]: + await self._fill_async() + return json.loads(self.buffer) + + async def _fill_async(self) -> Coroutine[Any, Any, None]: + if self.buffer is None: + data = await self.reader.readexactly(1) + if data: + self.type = MessageType(int.from_bytes( + data, byteorder='big', signed=True)) + data = await self.reader.readexactly(4) + data_len = int.from_bytes(data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.session_id = data.decode("utf-8") + if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): + data = await self.reader.readexactly(4) + data_len = int.from_bytes( + data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.buffer = data + + async def write_result_async(self, cms: dict, message_type: 'MessageType'): + try: + self.writer.write(message_type.value.to_bytes(1, 'big')) + data = self.session_id.encode() + data_length_bytes = len(data).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(data) + if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): + result_bytes = json.dumps( + cms, ensure_ascii=False).encode("utf-8") + data_length_bytes = len(result_bytes).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(result_bytes) + await self.writer.drain() + except asyncio.CancelledError: + pass + + async def set_result_async(self, cms: dict): + await self.write_result_async(cms, self.type) diff --git a/bclib/listener/web_message.py b/bclib/listener/web_message.py index 979de53..769cacd 100644 --- a/bclib/listener/web_message.py +++ b/bclib/listener/web_message.py @@ -1,20 +1,188 @@ -from typing import Any, Coroutine, Optional, Union +import base64 +import pathlib +import cgi +import io +import datetime from aiohttp import web +from urllib.parse import unquote, parse_qs +from typing import Any, Coroutine, Optional, Union +from multidict import MultiDict from aiohttp.web_response import ContentCoding +from bclib.utility.response_types import ResponseTypes +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.message import Message -from bclib.listener.message_type import MessageType class WebMessage(Message): - def __init__(self, request: 'web.Request',sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - super().__init__(sessionId,messageType,buffer) + _id = 0 + def __init__(self, request: 'web.Request') -> None: + super().__init__() self.__request = request - self.Response = None + self.Response: web.Response = None + + async def get_json_async(self) -> Coroutine[Any, Any, dict]: + return await self.create_cms_async(self.__request) + + async def set_result_async(self,cms:dict): + cms_cms = cms[HttpBaseDataType.CMS] + cms_cms_webserver = cms_cms[HttpBaseDataType.WEB_SERVER] + index = cms_cms_webserver[HttpBaseDataName.INDEX] + header_code: str = cms_cms_webserver[HttpBaseDataName.HEADER_CODE] + mime = cms_cms[HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] + headers = MultiDict() + if HttpBaseDataName.HTTP in cms_cms: + http: dict = cms_cms[HttpBaseDataName.HTTP] + for key, value in http.items(): + if isinstance(value, list): + for item in value: + headers.add(key, item) + else: + headers.add(key, value) + headers.add("Content-Type", mime) + if index == ResponseTypes.STATIC_FILE: + try: + path = pathlib.Path(cms_cms_webserver[HttpBaseDataName.FILE_PATH]) + path.stat() + self.Response = web.FileResponse( + path=path, + chunk_size=256*1024, + status=int(header_code.split(' ')[0]), + headers=headers + ) + except FileNotFoundError: + self.Response = web.Response( + status=404, + reason="File not found" + ) + else: + self.Response = web.Response( + status=int(header_code.split(' ')[0]), + headers=headers + ) + if HttpBaseDataName.CONTENT in cms_cms: + value = cms_cms[HttpBaseDataName.CONTENT] + self.Response.text =value if value is None or isinstance(value,str) else str(value) + else: + raw_blob_content = cms_cms[HttpBaseDataName.BLOB_CONTENT] + #TODO:Check for remove extra encoding + self.Response.body = base64.b64decode(raw_blob_content.encode("utf-8")) + + @staticmethod + async def create_cms_async(request: 'web.Request') -> dict: + cms_object = dict() + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.METHODE, request.method.lower()) + raw_url = unquote(request.path_qs)[1:] + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.RAW_URL, raw_url) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.URL, request.path[1:]) + WebMessage.__add_query_string(request.query, cms_object) + for key, value in request.headers.items(): + field_name = key.strip().lower() + if field_name == HttpBaseDataName.COOKIE: + WebMessage.__add_cookie(value, cms_object) + elif field_name == HttpBaseDataName.HOST: + WebMessage.__add_host(value, raw_url, cms_object) + else: + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + field_name, str(value).strip()) + WebMessage.__add_server_data(cms_object, request) + await WebMessage.__add_body_async(cms_object, request) + return {"cms": cms_object} + + @staticmethod + async def __add_body_async(cms_object: dict, request: 'web.Request'): + content_len_str = request.headers.get('Content-Length') + if content_len_str or request.can_read_body: + raw_body = await request.read() + body = raw_body.decode('utf-8') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.BODY, body) + content_type: str = request.headers.get( + "content-type") + if content_type and content_type.find("application/json") < 0: + if content_type.find("multipart/form-data") >= 0: + _, content_type_value_params = cgi.parse_header( + content_type) + content_type_value_params['boundary'] = bytes( + content_type_value_params['boundary'], "utf-8") + if content_len_str is not None: + content_type_value_params['CONTENT-LENGTH'] = int( + content_len_str) + with io.BytesIO(raw_body) as stream: + fields = cgi.parse_multipart( + stream, content_type_value_params) + for key, value in fields.items(): + WebMessage.__add_header(cms_object, + HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) + else: + for key, value in parse_qs(body).items(): + WebMessage.__add_header( + cms_object, HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) + + @staticmethod + def __add_server_data(cms_object: dict, request: 'web.Request'): + WebMessage._id += 1 + now = datetime.datetime.now() + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE, now.strftime("%d/%m/%Y")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.TIME, now.strftime("%H:%M")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE2, now.strftime("%Y%m%d")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.TIME2, now.strftime("%H%M%S")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE3, now.strftime("%Y.%m.%d")) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.REQUEST_ID, str(WebMessage._id)) + host_parts = request.host.split(':') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST_IP, host_parts[0]) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST_PORT, host_parts[1] if len(host_parts) > 1 else "80") # edit + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.CLIENT_IP, str(request.remote)) + + @staticmethod + def __add_query_string(query: dict, cms_object: dict) -> None: + for key, value in query.items(): + WebMessage.__add_header( + cms_object, HttpBaseDataType.QUERY, key, value) + + @staticmethod + def __add_cookie(raw_header_value: str, cms_object) -> None: + for item in raw_header_value.split(';'): + parts = item.split('=') + if len(parts) == 2: + WebMessage.__add_header(cms_object, HttpBaseDataName.COOKIE, + parts[0].strip(), parts[1].strip()) + @staticmethod + def __add_host(row_host: str, row_url: str, cms_object) -> None: + host_parts = row_host.split(':') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST, host_parts[0]) + if len(host_parts) == 2: + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.PORT, host_parts[1]) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.FULL_URL, f"{row_host}/{row_url}") - def create_response_message(self, session_id: str, buffer: bytes) -> "Message": - ret_val = WebMessage (self.__request, session_id,MessageType.AD_HOC,buffer) - ret_val.Response = self.Response - return ret_val + @staticmethod + def __add_header(cms_object: dict, value_type: str, value_name: str, value: str) -> None: + if value_type not in cms_object: + cms_object[value_type] = dict() + type_node = cms_object[value_type] + if value_name in type_node: + name_node = type_node[value_name] + if isinstance(name_node, list): + name_node.append(value) + else: + type_node[value_name] = [name_node, value] + else: + type_node[value_name] = value async def start_stream_response_async(self,status: int = 200, reason: Optional[str] = 'OK', diff --git a/bclib/listener/windows_named_pipe_listener.py b/bclib/listener/windows_named_pipe_listener.py deleted file mode 100644 index c56f1b2..0000000 --- a/bclib/listener/windows_named_pipe_listener.py +++ /dev/null @@ -1,109 +0,0 @@ -import asyncio -from typing import Callable, Coroutine -from ..listener.message import Message -from bclib.utility import WindowsNamedPipeHelper - - -class WindowsNamedPipeListener: - """""" - - def __init__(self, pipe_name: str, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): - self.pipe_name = pipe_name - self.on_message_receive = on_message_receive_call_back - self.__writer_pipe = None - self.__reader_pipe = None - self.__event_loop: asyncio.AbstractEventLoop = None - - async def __connect_writer_pipe_async(self): - import win32pipe - import pywintypes - try: - name = F"{self.pipe_name}/writer" - self.__writer_pipe = win32pipe.CreateNamedPipe(name, win32pipe.PIPE_ACCESS_OUTBOUND, - win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, - 1, 65536, 65536, 0, None) - print( - f"Writer named pipe '{name}' is created. Waiting for reader client to connect...") - await WindowsNamedPipeHelper.wait_for_client_connect_async(self.__writer_pipe, self.__event_loop) - print( - f"Reader client is connected to '{name}'...") - except pywintypes.error as e: # pylint: disable=maybe-no-member - self.__writer_pipe = None - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe. {repr(e)}") - else: - print(f"Named Pipe error code {e.args[0]}. {repr(e)}") - raise - except Exception as ex: - self.__writer_pipe = None - print(f"Error in create writer named pipe. {repr(ex)}") - raise - - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - - async def send_message_async(self, message: Message) -> bool: - try_count = 0 - send = False - while not send: - try: - if self.__writer_pipe is None: - await self.__connect_writer_pipe_async() - WindowsNamedPipeHelper.write_to_named_pipe( - message, self.__writer_pipe) - send = True - except asyncio.CancelledError: - break - except Exception as ex: - try_count = try_count+1 - self.__writer_pipe = None - print(f"Error in send message {ex}") - if try_count > 3: - break - await asyncio.sleep(.5) - return send - - def initialize_task(self, loop: asyncio.AbstractEventLoop): - self.__event_loop = loop - - async def reader_loop_async(): - import win32pipe - import win32file - import pywintypes - while True: - try: - name = F"{self.pipe_name}/reader" - self.__reader_pipe = win32pipe.CreateNamedPipe(name, win32pipe.PIPE_ACCESS_INBOUND, - win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, - 1, 65536, 65536, 0, None) - print( - f"Reader named pipe '{name}' is created. Waiting for writer client to connect...") - await WindowsNamedPipeHelper.wait_for_client_connect_async(self.__reader_pipe, self.__event_loop) - print( - f"Writer client connect to '{name}'...") - while True: - message = await WindowsNamedPipeHelper.read_from_named_pipe_async( - self.__reader_pipe, self.__event_loop) - if message: - self.__event_loop.create_task( - self.__process_message_async(message)) - except asyncio.CancelledError: - print('Edge named pipe server stopped.!') - break - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No reader named pipe. {repr(e)}") - elif e.args[0] == 109: # ERROR_BROKEN_PIPE - print(f"Reader named pipe is broken. {repr(e)}") - else: - print( - f"Reader named pipe error code {e.args[0]}. {repr(e)}") - finally: - # Disconnect the named pipe - win32pipe.DisconnectNamedPipe(self.__reader_pipe) - # CLose the named pipe - win32file.CloseHandle(self.__reader_pipe) - self.__event_loop.create_task(reader_loop_async()) - self.__event_loop.create_task(self.__connect_writer_pipe_async()) diff --git a/bclib/logger/__init__.py b/bclib/logger/__init__.py index 7403b8b..e69de29 100644 --- a/bclib/logger/__init__.py +++ b/bclib/logger/__init__.py @@ -1,3 +0,0 @@ -from bclib.logger.ilogger import ILogger -from bclib.logger.logger_factory import LoggerFactory -from bclib.logger.log_object import LogObject \ No newline at end of file diff --git a/bclib/logger/exchange_rabbit_schema_base_logger.py b/bclib/logger/exchange_rabbit_schema_base_logger.py index 963eb68..bcfc185 100644 --- a/bclib/logger/exchange_rabbit_schema_base_logger.py +++ b/bclib/logger/exchange_rabbit_schema_base_logger.py @@ -1,6 +1,6 @@ import asyncio import json -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx @@ -28,5 +28,5 @@ def send_to_rabbit(options): channel.basic_publish( exchange='', routing_key=queue, body=json.dumps(schema, ensure_ascii=False)) loop = asyncio.get_running_loop() - future = loop.run_in_executor(None, send_to_rabbit, self.options) + future = loop.run_in_executor(None, send_to_rabbit, self.options.logger) await future diff --git a/bclib/logger/ilogger.py b/bclib/logger/ilogger.py index d6813a2..abca7a1 100644 --- a/bclib/logger/ilogger.py +++ b/bclib/logger/ilogger.py @@ -1,10 +1,24 @@ from abc import ABC, abstractmethod -from .log_object import LogObject + +import traceback from typing import Optional +from bclib.utility import DictEx +from bclib.logger.log_object import LogObject + class ILogger(ABC): """Base class for logger""" + def __init__(self, options: 'DictEx'): + self.options = options + self.name = options["name"] if options.has("name") else None + self.__log_name = f"{self.name}: " if self.name else '' + self._log_error: bool = self.options.log_error if self.options.has( + "log_error") else False + self.__log_request: bool = self.options.log_request if self.options.has( + "log_request") else True + print(f'{self.__class__.__name__} start logging') + @abstractmethod async def log_async(self, log_object: LogObject): """log data async""" @@ -12,3 +26,11 @@ async def log_async(self, log_object: LogObject): def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: """New object log""" return LogObject(schema_name, routing_key, **kwargs) + + def log_request(self, message: 'str'): + if (self.__log_request): + print(self.__log_name, 'LOG', message) + + def log_error(self, error: 'Exception'): + print(self.__log_name, 'ERROR', str(error)) + traceback.print_exc() diff --git a/bclib/logger/log_schema.py b/bclib/logger/log_schema.py index 1ae45fa..2cfe7a7 100644 --- a/bclib/logger/log_schema.py +++ b/bclib/logger/log_schema.py @@ -1,5 +1,6 @@ from typing import Any, List, Dict, Tuple + class LogSchema: def __init__(self, schema: Dict[str, Any]) -> None: self.schema_name = schema["schemaName"] @@ -11,13 +12,13 @@ def __init__(self, schema: Dict[str, Any]) -> None: [ ( q["title"], ( - q["prpId"], + q["prpId"], bool(q.get("multi", False)), len(q["parts"]), - q["TypeID"] if "TypeID" in q else 0, + q["TypeID"] if "TypeID" in q else 0, q["source"] if "source" in q else None ) - ) + ) for q in schema["questions"] ] ) @@ -74,7 +75,6 @@ def get_answer(self, params: Dict[str, List[List]]): "TypeID": typeid, "answers": prp_answers }) - except: pass diff --git a/bclib/logger/logger_factory.py b/bclib/logger/logger_factory.py index 97e7034..b3ea0df 100644 --- a/bclib/logger/logger_factory.py +++ b/bclib/logger/logger_factory.py @@ -1,17 +1,18 @@ +from typing import Optional from bclib.utility import DictEx -from ..logger.rabbit_schema_base_logger import RabbitSchemaBaseLogger -from ..logger.restful_schema_base_logger import RESTfulSchemaBaseLogger -from ..logger.no_logger import NoLogger -from ..logger.ilogger import ILogger +from bclib.logger.rabbit_schema_base_logger import RabbitSchemaBaseLogger +from bclib.logger.restful_schema_base_logger import RESTfulSchemaBaseLogger +from bclib.logger.no_logger import NoLogger +from bclib.logger.ilogger import ILogger class LoggerFactory: @staticmethod def create(options: DictEx) -> ILogger: - logger: ILogger = None + logger: Optional[ILogger] = None if not options.has("logger"): - logger = NoLogger() + logger = NoLogger(options) else: logger_option: DictEx = options.logger if not logger_option.has('type'): @@ -19,11 +20,10 @@ def create(options: DictEx) -> ILogger: else: logger_type = logger_option.type.lower() if logger_type == 'schema.restful': - logger = RESTfulSchemaBaseLogger(logger_option) + logger = RESTfulSchemaBaseLogger(options) elif logger_type == "schema.rabbit": - logger = RabbitSchemaBaseLogger(logger_option) + logger = RabbitSchemaBaseLogger(options) else: raise Exception( f"Type '{logger_type}' not support for logger") - print(f'{logger.__class__.__name__} start logging') return logger diff --git a/bclib/logger/no_logger.py b/bclib/logger/no_logger.py index b5474dc..3e236a3 100644 --- a/bclib/logger/no_logger.py +++ b/bclib/logger/no_logger.py @@ -1,9 +1,13 @@ +from bclib.utility import DictEx from bclib.logger.log_object import LogObject -from ..logger.ilogger import ILogger +from bclib.logger.ilogger import ILogger class NoLogger(ILogger): """class for no logging""" + def __init__(self, options: 'DictEx'): + super().__init__(options) + async def log_async(self, log_object: LogObject): - """log data async""" \ No newline at end of file + """log data async""" diff --git a/bclib/logger/rabbit_schema_base_logger.py b/bclib/logger/rabbit_schema_base_logger.py index 982332b..0b0493f 100644 --- a/bclib/logger/rabbit_schema_base_logger.py +++ b/bclib/logger/rabbit_schema_base_logger.py @@ -1,26 +1,28 @@ import asyncio import json from typing import Optional -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx class RabbitSchemaBaseLogger(SchemaBaseLogger): - def __init__(self, options: DictEx) -> None: + def __init__(self, options: 'DictEx') -> None: super().__init__(options) - if "connection" not in options: + if "connection" not in options.logger: raise Exception("connection not set in logger option.") - self.__connection_options = options.connection + self.__connection_options = options.logger.connection if "url" not in self.__connection_options: raise Exception("url not set in connection option.") if "queue" in self.__connection_options and "exchange" in self.__connection_options: raise Exception("'queue' not acceptable when 'exchange' is set") elif "queue" not in self.__connection_options and "exchange" not in self.__connection_options: - raise Exception("'exchange' or 'queue' must be set in connection option") - + raise Exception( + "'exchange' or 'queue' must be set in connection option") + async def _save_schema_async(self, schema: dict, routing_key: Optional[str] = None): if routing_key is not None and self.__connection_options.queue is not None: - raise Exception("'routing key' is not acceptable when 'queue' is in options") + raise Exception( + "'routing key' is not acceptable when 'queue' is in options") def send_to_rabbit(): import pika @@ -40,8 +42,8 @@ def send_to_rabbit(): auto_delete=self.__connection_options.auto_delete or False ) channel.basic_publish( - exchange=self.__connection_options.exchange or "", - routing_key=routing_key or queue or "", + exchange=self.__connection_options.exchange or "", + routing_key=routing_key or queue or "", body=json.dumps(schema, ensure_ascii=False), properties=pika.BasicProperties( content_type="application/json", diff --git a/bclib/logger/restful_schema_base_logger.py b/bclib/logger/restful_schema_base_logger.py index b6b6f9d..0a99073 100644 --- a/bclib/logger/restful_schema_base_logger.py +++ b/bclib/logger/restful_schema_base_logger.py @@ -1,14 +1,14 @@ -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx class RESTfulSchemaBaseLogger(SchemaBaseLogger): - def __init__(self, options: DictEx) -> None: + def __init__(self, options: 'DictEx') -> None: super().__init__(options) - if options.has("url"): - self.__post_url = options.url - elif options.has("post_url"): - self.__post_url = options.post_url + if options.logger.has("url"): + self.__post_url = options.logger.url + elif options.logger.has("post_url"): + self.__post_url = options.logger.post_url else: raise Exception( "url part of schema logger not set. set 'url' or 'post_url'") diff --git a/bclib/logger/schema_base_logger.py b/bclib/logger/schema_base_logger.py index cc2f5ba..814d832 100644 --- a/bclib/logger/schema_base_logger.py +++ b/bclib/logger/schema_base_logger.py @@ -3,19 +3,18 @@ from bclib.logger.log_object import LogObject from bclib.utility import DictEx -from ..logger.log_schema import LogSchema -from ..logger.ilogger import ILogger +from bclib.logger.log_schema import LogSchema +from bclib.logger.ilogger import ILogger class SchemaBaseLogger(ILogger): - def __init__(self, options: DictEx) -> None: - super().__init__() - self.options = options - if options.has("url"): - self.__get_url = options.url - elif options.has("get_url"): - self.__get_url = options.get_url + def __init__(self, options: 'DictEx') -> None: + super().__init__(options) + if options.logger.has("url"): + self.__get_url = options.logger.url + elif options.logger.has("get_url"): + self.__get_url = options.logger.get_url else: raise Exception( "url part of schema logger not set. set 'url' or 'get_url'") diff --git a/bclib/parser/__init__.py b/bclib/parser/__init__.py index 9086f16..9974d41 100644 --- a/bclib/parser/__init__.py +++ b/bclib/parser/__init__.py @@ -3,5 +3,5 @@ from bclib.parser.answer import Answer, UserActionTypes, UserAction -def ParseAnswer(json: 'str|Any') -> Answer: +def ParseAnswer(json: 'str|Any') -> 'Answer': return Answer(json) diff --git a/bclib/parser/answer/answer.py b/bclib/parser/answer/answer.py index 8796130..d741fbd 100644 --- a/bclib/parser/answer/answer.py +++ b/bclib/parser/answer/answer.py @@ -5,23 +5,23 @@ from bclib.parser.answer.storage_data import StorageData from bclib.parser.answer.question_data import QuestionData from bclib import parser -from bclib.db_manager import RESTfulConnection +from bclib.db_manager.restful_connection import RESTfulConnection from bclib.parser.answer.validators import Validator from bclib.utility import DictEx -from ..answer.user_action_types import UserActionTypes -from ..answer.user_action import UserAction -import asyncio +from bclib.parser.answer.user_action_types import UserActionTypes +from bclib.parser.answer.user_action import UserAction + class Answer: """BasisJsonParser is a tool to parse basis_core components json objects. This tool is developed based on basis_core key and values.""" - def __init__(self, data: 'str|Any', api_url: 'str' = None, check_validation:"bool"= False): + def __init__(self, data: 'str|Any', api_url: 'str' = None, check_validation: "bool" = False): self.json = json.loads(data) if isinstance(data, str) else data self.__answer_list: 'list[UserAction]' = None self.__api_connection = RESTfulConnection( api_url) if api_url else None - self.check_validation = check_validation + self.check_validation = check_validation async def __fill_answer_list_async(self): self.__answer_list = list() @@ -32,13 +32,16 @@ async def __fill_answer_list_async(self): if action_type.value in list(data.keys()): prp_id = data['propId'] if action_type != UserActionTypes.ANSWERS else data["prpId"] for actions in data[action_type.value]: - prp_value_id = actions['id'] if 'id' in actions.keys() else None + prp_value_id = actions['id'] if 'id' in actions.keys( + ) else None if 'parts' in actions.keys(): for parts in actions['parts']: internal_prp_value_id = internal_prp_value_index - part_number = parts['part'] if "part" in parts.keys() else None + part_number = parts['part'] if "part" in parts.keys( + ) else None for values in parts['values']: - value_id = values['id'] if "id" in values.keys() else None + value_id = values['id'] if "id" in values.keys( + ) else None value = values['value'] answer = parser.ParseAnswer( values["answer"]) if 'answer' in values.keys() else None @@ -70,7 +73,8 @@ async def __enrich_data_async(self): for validations in parts.parts ] questions_info.append( - QuestionData(parts.prpId, parts.OwnerID, parts.TypeID if "TypeID" in parts else parts.typeid, parts.wordId, enriched_data_list) + QuestionData( + parts.prpId, parts.OwnerID, parts.TypeID if "TypeID" in parts else parts.typeid, parts.wordId, enriched_data_list) ) if len(questions_info) > 0: for values in self.__answer_list: @@ -88,23 +92,27 @@ async def __enrich_data_async(self): values.table = storage_data.table values.field = storage_data.field if self.check_validation and values.action != UserActionTypes.DELETED: - status, message = Validator.check_validators(enriched_data.validators, values.value) + status, message = Validator.check_validators( + enriched_data.validators, values.value) values.validation_status = status values.validation_message = message - - def __enrich_data(self, validations:DictEx) -> 'EnrichedData': + + def __enrich_data(self, validations: DictEx) -> 'EnrichedData': part_id = validations.part data_type = self.__set_data_type(validations) val_val = validations.validations - storage_data = self.__set_storage_data(val_val) if isinstance(val_val, dict) else None - validators = val_val if self.check_validation and isinstance(validations.validations, dict) else {} + storage_data = self.__set_storage_data( + val_val) if isinstance(val_val, dict) else None + validators = val_val if self.check_validation and isinstance( + validations.validations, dict) else {} return EnrichedData(part_id, data_type, storage_data, validators) - def __set_data_type(self, validations:DictEx) -> 'str': + def __set_data_type(self, validations: DictEx) -> 'str': has_link = True if validations.link else False val_val = validations.validations - data_type = val_val["dataType"] if isinstance(val_val, dict) and "dataType" in val_val else None - + data_type = val_val["dataType"] if isinstance( + val_val, dict) and "dataType" in val_val else None + return self.__data_type_checker(validations.viewType, data_type, has_link) def __data_type_checker(self, view_type: str, datatype: str = None, has_link: bool = None): @@ -137,13 +145,13 @@ def __data_type_checker(self, view_type: str, datatype: str = None, has_link: bo else: result = "None" return result - - def __set_storage_data(self, val_val:"dict") -> "StorageData": + + def __set_storage_data(self, val_val: "dict") -> "StorageData": database = val_val["database"] if "database" in val_val else None table = val_val["table"] if "table" in val_val else None field = val_val["field"] if "field" in val_val else None return StorageData(database, table, field) - + async def __get_action_async(self, prp_id_list: 'list[int]', action_list: 'list[UserActionTypes]', part_list: 'list[int]', is_file: "bool" = None, predicate: 'Callable[[UserAction],bool]' = None) -> 'list[UserAction]': ret_val: 'list[UserAction]' = None if self.__answer_list is None: diff --git a/bclib/parser/answer/question_data.py b/bclib/parser/answer/question_data.py index ec752b5..51ced11 100644 --- a/bclib/parser/answer/question_data.py +++ b/bclib/parser/answer/question_data.py @@ -1,9 +1,10 @@ -from ..answer.enriched_data import EnrichedData +from bclib.parser.answer.enriched_data import EnrichedData + class QuestionData: - def __init__(self, prpid:"int", ownerid:"int|None", typeid:"int|None", wordid:"int|None", enriched_data:"list[EnrichedData]") -> None: + def __init__(self, prpid: "int", ownerid: "int|None", typeid: "int|None", wordid: "int|None", enriched_data: "list[EnrichedData]") -> None: self.prpid = prpid self.ownerid = ownerid if ownerid is not None else 0 self.typeid = typeid self.wordid = wordid - self.enriched_data = enriched_data \ No newline at end of file + self.enriched_data = enriched_data diff --git a/bclib/parser/answer/user_action.py b/bclib/parser/answer/user_action.py index 4dc9880..bf12e45 100644 --- a/bclib/parser/answer/user_action.py +++ b/bclib/parser/answer/user_action.py @@ -1,10 +1,10 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from bclib.parser.answer.answer import Answer -from ..answer.user_action_types import UserActionTypes -from ..answer.file_user_action import FileUserAction -from ..answer.date_user_action import DateUserAction -from ..answer.time_user_action import TimeUserAction +from bclib.parser.answer.user_action_types import UserActionTypes +from bclib.parser.answer.file_user_action import FileUserAction +from bclib.parser.answer.date_user_action import DateUserAction +from bclib.parser.answer.time_user_action import TimeUserAction class UserAction: @@ -22,19 +22,19 @@ def __init__(self, prp_id: 'int', action: 'UserActionTypes', prp_value_id: 'int' self.multi = multi self.answer: 'Answer' = answer self.internal_prp_value_id: "int" = internal_prp_value_id - self.database:"str" = None - self.table:"str" = None - self.field:"str" = None - self.ownerid:"int" = 0 - self.typeid:"int" = None - self.wordid:"int" = None + self.database: "str" = None + self.table: "str" = None + self.field: "str" = None + self.ownerid: "int" = 0 + self.typeid: "int" = None + self.wordid: "int" = None self.validation_status: "bool" = True self.validation_message: "list[str]" = [] def as_tuple(self) -> tuple: return ( self.prp_id, self.action.value, self.prp_value_id, self.internal_prp_value_id, self.value_id, - self.value, self.part, self.datatype, self.database, self.table, self.field, self.multi, + self.value, self.part, self.datatype, self.database, self.table, self.field, self.multi, self.ownerid, self.typeid, self.wordid, self.answer, self.validation_status, self.validation_message ) @@ -62,7 +62,7 @@ def as_dict(self) -> dict: def is_date_useraction(self): return self.datatype == "datevalue" - + def as_date_useraction(self): if self.is_date_useraction(): value = self.value @@ -88,7 +88,7 @@ def as_time_useraction(self): value["time"], int(value["timeid"]) ) - + def is_file_content(self): return self.datatype == "files" diff --git a/bclib/parser/html/html_parser_ex.py b/bclib/parser/html/html_parser_ex.py index 610a4dc..2c8758e 100644 --- a/bclib/parser/html/html_parser_ex.py +++ b/bclib/parser/html/html_parser_ex.py @@ -2,7 +2,7 @@ from html.parser import HTMLParser from typing import Any from bclib.utility import DictEx -from ..html.html_tag import HtmlTag +from bclib.parser.html.html_tag import HtmlTag class HtmlParserEx(HTMLParser): diff --git a/bclib/predicate/all.py b/bclib/predicate/all.py index 782004b..71d77e8 100644 --- a/bclib/predicate/all.py +++ b/bclib/predicate/all.py @@ -1,5 +1,7 @@ -from ..predicate.predicate import Predicate -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class All (Predicate): @@ -9,7 +11,7 @@ def __init__(self, *predicate: 'Predicate') -> None: super().__init__(None) self.__predicate_list = predicate - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': is_ok = True try: for predicate in self.__predicate_list: diff --git a/bclib/predicate/any.py b/bclib/predicate/any.py index c8651f7..5c22b61 100644 --- a/bclib/predicate/any.py +++ b/bclib/predicate/any.py @@ -1,5 +1,7 @@ -from ..predicate.predicate import Predicate -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Any (Predicate): @@ -9,7 +11,7 @@ def __init__(self, *predicate: 'Predicate') -> None: super().__init__(None) self.__predicate_list = predicate - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': is_ok = False try: for predicate in self.__predicate_list: diff --git a/bclib/predicate/between.py b/bclib/predicate/between.py index ddbe5a2..8a3ab00 100644 --- a/bclib/predicate/between.py +++ b/bclib/predicate/between.py @@ -1,5 +1,7 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Between(Predicate): @@ -10,9 +12,9 @@ def __init__(self, expression: str, min_value: int, max_value: int): self.__min_value = min_value self.__max_value = max_value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__min_value < int(value) < self.__max_value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/callback.py b/bclib/predicate/callback.py index 42d08c5..3bd79c4 100644 --- a/bclib/predicate/callback.py +++ b/bclib/predicate/callback.py @@ -1,17 +1,19 @@ from bclib.exception import ShortCircuitErr -from ..predicate.predicate import Predicate -from typing import Callable, Coroutine -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import Callable, Coroutine, TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Callback (Predicate): - """Create callback base cheking predicate""" + """Create callback base checking predicate""" def __init__(self, callback: 'Callable[[Context],Coroutine[bool]]') -> None: super().__init__(None) self.__callback = callback - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: return await self.__callback(context) except ShortCircuitErr: diff --git a/bclib/predicate/equal.py b/bclib/predicate/equal.py index 9f462ed..3db596b 100644 --- a/bclib/predicate/equal.py +++ b/bclib/predicate/equal.py @@ -1,5 +1,7 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Equal (Predicate): @@ -9,9 +11,9 @@ def __init__(self, expression, value) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value == value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/greater_than.py b/bclib/predicate/greater_than.py index 8921fa6..ef9b7c5 100644 --- a/bclib/predicate/greater_than.py +++ b/bclib/predicate/greater_than.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class GreaterThan (Predicate): - """Create greater than cheking predicate""" + """Create greater than checking predicate""" def __init__(self, expression: str, value: int) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value < value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/greater_than_equal.py b/bclib/predicate/greater_than_equal.py index 797d1db..e5c2216 100644 --- a/bclib/predicate/greater_than_equal.py +++ b/bclib/predicate/greater_than_equal.py @@ -1,17 +1,19 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class GreaterThanEqual (Predicate): - """Create greater than equal cheking predicate""" + """Create greater than equal checking predicate""" def __init__(self, expression: str, value: int) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value <= value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/has_value.py b/bclib/predicate/has_value.py index c93ebb6..d963e9b 100644 --- a/bclib/predicate/has_value.py +++ b/bclib/predicate/has_value.py @@ -1,17 +1,19 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate -from typing import Any +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class HasValue (Predicate): - """Create has value cheking predicate""" + """Create has value checking predicate""" def __init__(self, expression: str) -> None: super().__init__(expression) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return False if not value or value.isspace() else True except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/in_list.py b/bclib/predicate/in_list.py index 9e799b2..9af8895 100644 --- a/bclib/predicate/in_list.py +++ b/bclib/predicate/in_list.py @@ -1,18 +1,20 @@ -from typing import Any -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from bclib.context.context import Context class InList(Predicate): - """Create list cheking predicate""" + """Create list checking predicate""" def __init__(self, expression: str, *items: Any) -> None: super().__init__(expression) self.__items = [*items] - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return value in self.__items except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/less_than.py b/bclib/predicate/less_than.py index 179daf3..d3937a7 100644 --- a/bclib/predicate/less_than.py +++ b/bclib/predicate/less_than.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class LessThan (Predicate): - """Create less than cheking predicate""" + """Create less than checking predicate""" - def __init__(self, expression: str, value: int) -> None: + def __init__(self, expression: 'str', value: 'int') -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value > value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/less_than_equal.py b/bclib/predicate/less_than_equal.py index 0edde36..c8d9345 100644 --- a/bclib/predicate/less_than_equal.py +++ b/bclib/predicate/less_than_equal.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class LessThanEqual (Predicate): - """Create less than and equal cheking predicate""" + """Create less than and equal checking predicate""" - def __init__(self, expression: str, value: int) -> None: + def __init__(self, expression: 'str', value: 'int') -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value >= value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/match.py b/bclib/predicate/match.py index 4c22a6a..ca552c1 100644 --- a/bclib/predicate/match.py +++ b/bclib/predicate/match.py @@ -1,18 +1,21 @@ import re -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Match (Predicate): - """Create regex matching cheking predicate""" + """Create regex matching checking predicate""" - def __init__(self, expression: str, value: str) -> None: + def __init__(self, expression: 'str', value: 'str') -> None: super().__init__(expression) self.__compiled_regex = re.compile(value) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__compiled_regex.match(str(value)) != None except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/not_equal.py b/bclib/predicate/not_equal.py index 46e67a7..28cd371 100644 --- a/bclib/predicate/not_equal.py +++ b/bclib/predicate/not_equal.py @@ -1,18 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate -from typing import Any +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from bclib.context.context import Context class NotEqual (Predicate): - """Create not equality cheking predicate""" + """Create not equality checking predicate""" def __init__(self, expression: str, value: Any) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value != value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/predicate.py b/bclib/predicate/predicate.py index 24eadfc..326877e 100644 --- a/bclib/predicate/predicate.py +++ b/bclib/predicate/predicate.py @@ -1,14 +1,17 @@ from abc import ABC, abstractmethod -from bclib.context import Context +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Predicate(ABC): """Base class for predicate""" - def __init__(self, expression: str) -> None: + def __init__(self, expression: 'str') -> 'None': super().__init__() - self.exprossion = expression + self.expression = expression @abstractmethod - async def check_async(self, context: Context) -> bool: - """Applay cheking for predicate""" + async def check_async(self, context: 'Context') -> 'bool': + """Apply checking for predicate""" diff --git a/bclib/predicate/url.py b/bclib/predicate/url.py index 96f51ba..57356a9 100644 --- a/bclib/predicate/url.py +++ b/bclib/predicate/url.py @@ -1,17 +1,20 @@ from types import FunctionType -from bclib.context import Context from bclib.utility import DictEx -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Url (Predicate): - """Create Url cheking predicate""" + """Create Url checking predicate""" - def __init__(self, expression: str) -> None: + def __init__(self, expression: 'str') -> 'None': super().__init__(expression) self.__validator: FunctionType = Url.__generate_validator(expression) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: is_ok, url_parts = self.__validator(context.url) if is_ok and url_parts: @@ -60,7 +63,7 @@ def url_function(url): try: url_parts = url.split("/") if {" and ".join(where_part_list)}: - {','.join(segment_list)} = url_parts{"[0]" if len(segment_list)==1 else ""} + {','.join(segment_list)} = url_parts{"[0]" if len(segment_list) == 1 else ""} return (True,{{ {','.join(return_dict_property_names)} }}) else: return (False,None) diff --git a/bclib/utility/__init__.py b/bclib/utility/__init__.py index a689ee5..1e80c42 100644 --- a/bclib/utility/__init__.py +++ b/bclib/utility/__init__.py @@ -3,5 +3,3 @@ from bclib.utility.http_mime_types import HttpMimeTypes from bclib.utility.response_types import ResponseTypes from bclib.utility.http_headers import HttpHeaders -from bclib.utility.windows_named_pipe_helper import WindowsNamedPipeHelper -from bclib.utility.linux_named_pipe_helper import LinuxNamedPipeHelper diff --git a/bclib/utility/dict_ex.py b/bclib/utility/dict_ex.py index e971176..a984b5d 100644 --- a/bclib/utility/dict_ex.py +++ b/bclib/utility/dict_ex.py @@ -4,7 +4,7 @@ class DictEx(dict): """Extended version of dict class for dot.notation access to attributes""" - def __init__(self, *args, **kwargs): + def __init__(self, *args): super().__init__() for arg in args: if isinstance(arg, dict): diff --git a/bclib/utility/linux_named_pipe_helper.py b/bclib/utility/linux_named_pipe_helper.py deleted file mode 100644 index 768a71c..0000000 --- a/bclib/utility/linux_named_pipe_helper.py +++ /dev/null @@ -1,64 +0,0 @@ -import asyncio -from io import BufferedReader, BufferedWriter -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from bclib.listener.message import Message - - -class LinuxNamedPipeHelper: - - @staticmethod - def __read_from_named_pipe(named_pipe_handle: BufferedReader, future: asyncio.Future) -> None: - from bclib.listener import Message, MessageType - - try: - data = named_pipe_handle.read(1) - if data: - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = named_pipe_handle.read(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = named_pipe_handle.read(data_len) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = named_pipe_handle.read(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - parameter = named_pipe_handle.read( - data_len) - message = Message(session_id, message_type, parameter) - else: - message = None - future.get_loop().call_soon_threadsafe(future.set_result, message) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - - @staticmethod - async def read_from_named_pipe_async(named_pipe_handle: BufferedReader, loop: asyncio.AbstractEventLoop = None) -> 'Message': - if not loop: - loop = asyncio.get_event_loop() - future = loop.create_future() - loop.run_in_executor( - None, LinuxNamedPipeHelper.__read_from_named_pipe, named_pipe_handle, future) - return await future - - @staticmethod - def write_to_named_pipe(message: 'Message', named_pipe_handle: BufferedWriter): - from bclib.listener import MessageType - - named_pipe_handle.write(message.type.value.to_bytes(1, 'big')) - - data = message.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - named_pipe_handle.write(data_length_bytes) - named_pipe_handle.write(data) - - if message.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(message.buffer).to_bytes(4, 'big') - named_pipe_handle.write(data_length_bytes) - named_pipe_handle.write(message.buffer) - - named_pipe_handle.flush() diff --git a/bclib/utility/windows_named_pipe_helper.py b/bclib/utility/windows_named_pipe_helper.py deleted file mode 100644 index 4dce194..0000000 --- a/bclib/utility/windows_named_pipe_helper.py +++ /dev/null @@ -1,119 +0,0 @@ -import asyncio -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from bclib.listener.message import Message - - -class WindowsNamedPipeHelper: - - @staticmethod - def __check_read_error(error_code): - if error_code != 0: - raise Exception( - f"error in read from pip. error code = {error_code}") - - @staticmethod - def __check_write_error(error_code): - if error_code != 0: - raise Exception( - f"error in write in pip. error code = {error_code}") - - @staticmethod - def __read_from_named_pipe(named_pipe_handle: Any, future: asyncio.Future) -> None: - import win32file - from bclib.listener import Message, MessageType - - try: - error, data = win32file.ReadFile(named_pipe_handle, 1) - WindowsNamedPipeHelper.__check_read_error(error) - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - error, data = win32file.ReadFile(named_pipe_handle, 4) - WindowsNamedPipeHelper.__check_read_error(error) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - error, data = win32file.ReadFile(named_pipe_handle, data_len) - WindowsNamedPipeHelper.__check_read_error(error) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - error, data = win32file.ReadFile(named_pipe_handle, 4) - WindowsNamedPipeHelper.__check_read_error(error) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - error, parameter = win32file.ReadFile( - named_pipe_handle, data_len) - WindowsNamedPipeHelper.__check_read_error(error) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - else: - message = Message(session_id, message_type, parameter) - future.get_loop().call_soon_threadsafe(future.set_result, message) - - @staticmethod - async def read_from_named_pipe_async(named_pipe_handle: Any, loop: asyncio.AbstractEventLoop = None) -> 'Message': - if not loop: - loop = asyncio.get_event_loop() - future = loop.create_future() - loop.run_in_executor( - None, WindowsNamedPipeHelper.__read_from_named_pipe, named_pipe_handle, future) - return await future - - @staticmethod - def write_to_named_pipe(message: 'Message', named_pipe_handle: Any): - import win32file - import win32pipe - import pywintypes - from bclib.listener import MessageType - - try: - error, _ = win32file.WriteFile( - named_pipe_handle, message.type.value.to_bytes(1, 'big')) - WindowsNamedPipeHelper.__check_write_error(error) - data = message.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - error, _ = win32file.WriteFile( - named_pipe_handle, data_length_bytes) - WindowsNamedPipeHelper.__check_write_error(error) - error, _ = win32file.WriteFile(named_pipe_handle, data) - WindowsNamedPipeHelper.__check_write_error(error) - if message.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(message.buffer).to_bytes(4, 'big') - error, _ = win32file.WriteFile( - named_pipe_handle, data_length_bytes) - WindowsNamedPipeHelper.__check_write_error(error) - error, _ = win32file.WriteFile( - named_pipe_handle, message.buffer) - WindowsNamedPipeHelper.__check_write_error(error) - win32file.FlushFileBuffers(named_pipe_handle) - except pywintypes.error as e: # pylint: disable=maybe-no-member - # Disconnect the named pipe - win32pipe.DisconnectNamedPipe(named_pipe_handle) - # CLose the named pipe - win32file.CloseHandle(named_pipe_handle) - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe. {repr(e)}") - elif e.args[0] == 232: - print(f"The Pipe is being closed. {repr(e)}") - elif e.args[0] == 109: # ERROR_BROKEN_PIPE - print(f"Named Pipe is broken. {repr(e)}") - else: - print(f"Named Pipe error code {e.args[0]}. {repr(e)}") - raise - - @staticmethod - async def wait_for_client_connect_async(pipe_handler: any, loop: asyncio.AbstractEventLoop = None) -> None: - import win32pipe - future = loop.create_future() - - def process(pipe_handler: any, future: asyncio.Future): - try: - win32pipe.ConnectNamedPipe(pipe_handler, None) - future.get_loop().call_soon_threadsafe(future.set_result, True) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - if not loop: - loop = asyncio.get_event_loop() - loop.run_in_executor(None, process, pipe_handler, future) - await future diff --git a/requirements.tx b/requirements.tx new file mode 100644 index 0000000..0103311 --- /dev/null +++ b/requirements.tx @@ -0,0 +1,3 @@ +aiohttp==3.11.10 +dependency-injector==4.44.0 +legacy-cgi==2.6.1 diff --git a/setup.py b/setup.py index 4c212c5..327d744 100644 --- a/setup.py +++ b/setup.py @@ -21,14 +21,13 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - # install_requires=[ - # 'pika', - # 'requests', - # 'pymongo', - # 'pyodbc' - # ], - #package_dir={"": "basiscore"}, + install_requires=[ + 'aiohttp', + 'legacy-cgi', + 'dependency-injector' + ], + # package_dir={"": "basiscore"}, packages=setuptools.find_packages(exclude=["test", "app-env", ".vscode"]), - python_requires=">=3.7", + python_requires=">=3.13", setup_requires=['wheel'] ) diff --git a/test/all_test.py b/test/all_test.py index f14906f..7f201b4 100644 --- a/test/all_test.py +++ b/test/all_test.py @@ -185,7 +185,7 @@ def process_not_exist_message(context: edge.SocketContext): ChatRoom.process_message(context.message, None, None) -@ app.socket_action() +@app.socket_action() def process_all_other_message(context: edge.SocketContext): print("process_all_other_message") ChatRoom.process_message(context.message, diff --git a/test/dev_server/simple.py b/test/dev_server/simple.py index f18decc..0f693d7 100644 --- a/test/dev_server/simple.py +++ b/test/dev_server/simple.py @@ -1,13 +1,44 @@ +import asyncio +import random from bclib import edge +from bclib.edge import EdgeContainer +from dependency_injector import containers, providers +from dependency_injector.wiring import Provide, inject -if "options" not in dir(): - options = { - "server": "localhost:8080", - "router": "web" - } +def manage_resource1(): + print("init tasks 1") + yield + print("stop tasks 1") -app = edge.from_options(options) + +async def manage_resource2_async(): + await asyncio.sleep(.5) + print("init tasks 2") + yield None + await asyncio.sleep(.5) + print("stop tasks 2") + + +@containers.copy(EdgeContainer) +class AppContainer(EdgeContainer): + app_resource1 = providers.Resource(manage_resource1) + # app_resource2 = providers.Resource(manage_resource2_async) + object_provider = providers.Callable(lambda: random.randint(100, 999)) + object_val = providers.Callable(lambda: random.randint(100, 999)) + object_val2 = providers.Callable(lambda: random.randint(1000, 9999)) + + +container = AppContainer() +container.app_config.from_dict({ + "server": "localhost:8080", + "router": "web", + # "loop":asyncio.get_event_loop() + "cache": {"1": 22} +}) + + +app = edge.from_container(container) async def check_async(context: edge.RequestContext): @@ -20,8 +51,10 @@ def process_web_action(context: edge.WebContext): @app.web_action() -def process_default_web_action(context: edge.WebContext): - return "result from process_default_web_action" +@inject +def process_default_web_action(context: edge.WebContext, val=Provide["object_provider"], container: AppContainer = Provide["app_container"]): + return "result from process_default_web_action " + str(val or "?") + " " + str(context.dispatcher.container.object_val()) + " " + str(container.object_val2()) +container.wire(modules=[__name__]) app.listening() diff --git a/test/web/simple.py b/test/web/simple.py index 170618e..8e5e552 100644 --- a/test/web/simple.py +++ b/test/web/simple.py @@ -1,9 +1,11 @@ import asyncio from concurrent.futures import thread -from bclib import edge +# from bclib import edge import time +from bclib import edge + options = { "endpoint": "127.0.0.1:1025", @@ -12,8 +14,6 @@ app = edge.from_options(options) -print(app) - @app.web_action() async def process_web_remain_request(context: edge.WebContext): diff --git a/test/web_socket/list-data.py b/test/web_socket/list-data.py index 6ea9cac..9780f93 100644 --- a/test/web_socket/list-data.py +++ b/test/web_socket/list-data.py @@ -56,7 +56,7 @@ async def send_data_async(context: edge.SocketContext): } id += 1 print( - f'Send data to {context.message.session_id} in {datetime.datetime.now()}') + f'Send data to {context.message.session_id} in {datetime.datetime.now()}',type(context)) try: await context.send_object_async(data) except: # ConnectionError @@ -73,8 +73,8 @@ async def send_data_async(context: edge.SocketContext): ######## -@app.socket_action() -async def process_message_async(context: edge.SocketContext): +@app.endpoint_action() +async def process_message_async(context: edge.EndPointContext): print( f'message of type {context.message.type} come from {context.message.session_id} in {datetime.datetime.now()}') msg = await context.read_message_async() @@ -82,15 +82,15 @@ async def process_message_async(context: edge.SocketContext): f'message of type {msg.type} come from {msg.session_id} in {datetime.datetime.now()}') future = context.dispatcher.run_in_background( send_data_async, context) - while True: + while not future.done(): try: msg = await context.read_message_async() print( f'message of type {msg.type} come from {msg.session_id} in {datetime.datetime.now()}') if msg.type in [edge.MessageType.DISCONNECT, edge.MessageType.NOT_EXIST]: break - except: - print("connection closed!") + except Exception as ex: + print("connection closed!",ex) break future.cancel()