From 598a50f721cd9194fb6aa468e7501829dad7032b Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:49:08 +0330 Subject: [PATCH 01/15] adding DI --- bclib/cache/__init__.py | 3 +- bclib/cache/manager.py | 2 +- bclib/context/context.py | 10 ++++-- bclib/dispatcher/__init__.py | 1 + bclib/dispatcher/dev_server_dispatcher.py | 13 +++++-- bclib/dispatcher/dispatcher.py | 42 +++++++++++------------ bclib/dispatcher/endpoint_dispatcher.py | 12 +++++-- 7 files changed, 50 insertions(+), 33 deletions(-) diff --git a/bclib/cache/__init__.py b/bclib/cache/__init__.py index 5409366..1cf4414 100644 --- a/bclib/cache/__init__.py +++ b/bclib/cache/__init__.py @@ -1,2 +1,3 @@ from bclib.cache.manager import CacheManager -from bclib.cache.factory import CacheFactory \ No newline at end of file +from bclib.cache.factory import CacheFactory +from bclib.cache.cache_status import CacheStatus \ No newline at end of file diff --git a/bclib/cache/manager.py b/bclib/cache/manager.py index 363fd90..322e0cf 100644 --- a/bclib/cache/manager.py +++ b/bclib/cache/manager.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from bclib.utility import DictEx -from ..cache.cache_status import CacheStatus +from .cache_status import CacheStatus class CacheManager(ABC): def __init__(self, options:"DictEx") -> None: diff --git a/bclib/context/context.py b/bclib/context/context.py index e47923e..dfac6c7 100644 --- a/bclib/context/context.py +++ b/bclib/context/context.py @@ -1,6 +1,6 @@ from abc import ABC import traceback -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from bclib.exception import ShortCircuitErr @@ -10,7 +10,7 @@ import base64 if TYPE_CHECKING: - from .. import dispatcher + from bclib import dispatcher class Context(ABC): @@ -20,9 +20,13 @@ def __init__(self, dispatcher: 'dispatcher.IDispatcher') -> None: super().__init__() self.dispatcher = dispatcher self.url_segments: DictEx = None - self.url: str = None + self.url: Optional[str] = None self.is_adhoc = True + @property + def container(self): + return self.dispatcher.container + def open_sql_connection(self, key: str) -> SqlDb: return self.dispatcher.db_manager.open_sql_connection(key) diff --git a/bclib/dispatcher/__init__.py b/bclib/dispatcher/__init__.py index a149245..2f7b440 100644 --- a/bclib/dispatcher/__init__.py +++ b/bclib/dispatcher/__init__.py @@ -5,3 +5,4 @@ from bclib.dispatcher.routing_dispatcher import RoutingDispatcher from bclib.dispatcher.named_pipe_dispatcher import NamedPipeDispatcher from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher +from bclib.dispatcher.dispatcher_helper import DispatcherHelper diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index 5023d1d..484625a 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -1,11 +1,18 @@ import asyncio -from ..dispatcher.socket_dispatcher import RoutingDispatcher + +from dependency_injector import containers + +from bclib.logger import ILogger +from bclib.cache import CacheManager +from bclib.db_manager import DbManager +from bclib.utility import DictEx from bclib.listener import Endpoint, HttpListener +from .routing_dispatcher import RoutingDispatcher class DevServerDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container=container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger = logger, loop=loop) self.__listener = HttpListener( Endpoint(self.options.server), self._on_message_receive_async, diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index e7dd7c6..71bbbf5 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -3,37 +3,33 @@ import inspect from abc import ABC import signal -import sys import traceback -from typing import Callable, Any, Coroutine, Optional +from typing import Awaitable, Callable, Any, Coroutine, Optional from functools import wraps -from bclib.logger import ILogger, LoggerFactory, LogObject -from bclib.cache import CacheFactory +from dependency_injector import containers + +from bclib.logger import ILogger, LogObject +from bclib.cache import CacheManager from bclib.listener import RabbitBusListener from bclib.predicate import Predicate from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext, NamedPipeContext from bclib.db_manager import DbManager from bclib.utility import DictEx from bclib.exception import HandlerNotFoundErr -from ..dispatcher.callback_info import CallbackInfo +from .callback_info import CallbackInfo class Dispatcher(ABC): """Base class for dispatching request""" - def __init__(self, options: dict = None,loop:asyncio.AbstractEventLoop = None): - self.options = DictEx(options) + def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger', loop:'asyncio.AbstractEventLoop'): + self.options = options + self.container= container self.__look_up: 'dict[str, list[CallbackInfo]]' = dict() - cache_options = self.options.cache if "cache" in self.options else None - if loop is None and sys.platform == 'win32': - # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. - # Use Windows version of proactor event loop using IOCP - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - self.event_loop = asyncio.get_event_loop() if loop is None else loop - self.cache_manager = CacheFactory.create(cache_options) - self.db_manager = DbManager(self.options, self.event_loop) - self.__logger: ILogger = LoggerFactory.create(self.options) + self.event_loop: 'asyncio.AbstractEventLoop' = loop + self.cache_manager: 'CacheManager' = cache_manager + self.db_manager = db_manager + self.__logger: ILogger = logger self.log_error: bool = self.options.log_error if self.options.has( "log_error") else False self.log_request: bool = self.options.log_request if self.options.has( @@ -387,12 +383,13 @@ def initialize_task(self): for dispatcher in self.__rabbit_dispatcher: dispatcher.initialize_task(self.event_loop) - def listening(self, before_start: Coroutine=None, after_end: Coroutine=None,with_block:bool = True): + def listening(self, with_block:bool = True): """Start listening to request for process""" for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda sig, _: self.event_loop.stop()) - if before_start != None: - self.event_loop.run_until_complete(self.event_loop.create_task(before_start())) + init_process = self.container.init_resources() + if isinstance( init_process,Awaitable): + self.event_loop.run_until_complete( init_process) self.initialize_task() if with_block: self.event_loop.run_forever() @@ -401,8 +398,9 @@ def listening(self, before_start: Coroutine=None, after_end: Coroutine=None,with task.cancel() group = asyncio.gather(*tasks, return_exceptions=True) self.event_loop.run_until_complete(group) - if after_end != None: - self.event_loop.run_until_complete(self.event_loop.create_task(after_end())) + shutdown_process = self.container.shutdown_resources() + if isinstance( shutdown_process,Awaitable): + self.event_loop.run_until_complete(shutdown_process) self.event_loop.close() def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index da734ec..8d2fb55 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -1,12 +1,18 @@ import asyncio -from ..listener import Endpoint, ReceiveMessage +from dependency_injector import containers +from bclib.cache import CacheManager +from bclib.db_manager import DbManager +from bclib.logger import ILogger + +from bclib.utility import DictEx +from bclib.listener import Endpoint, ReceiveMessage from .routing_dispatcher import RoutingDispatcher class EndpointDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self, container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) self.__endpoint = Endpoint(self.options.endpoint) def initialize_task(self): From ff527a9f8709aa5371ed88df8ba7903e08da5f71 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:50:35 +0330 Subject: [PATCH 02/15] Adding DI part 2 --- bclib/dispatcher/idispatcher.py | 3 ++ bclib/dispatcher/named_pipe_dispatcher.py | 2 +- bclib/dispatcher/routing_dispatcher.py | 14 ++++--- bclib/dispatcher/socket_dispatcher.py | 14 +++++-- bclib/edge.py | 38 +++++++++++++++-- bclib/edge_container.py | 50 +++++++++++++++++++++++ bclib/utility/dict_ex.py | 2 +- test/dev_server/simple.py | 47 ++++++++++++++++----- 8 files changed, 144 insertions(+), 26 deletions(-) create mode 100644 bclib/edge_container.py diff --git a/bclib/dispatcher/idispatcher.py b/bclib/dispatcher/idispatcher.py index 2ba3a7f..eb69fe8 100644 --- a/bclib/dispatcher/idispatcher.py +++ b/bclib/dispatcher/idispatcher.py @@ -1,6 +1,7 @@ """Dispatcher base class module""" from abc import ABC, abstractmethod import asyncio +from dependency_injector import containers from typing import Callable, Any, TYPE_CHECKING, Coroutine, Optional from bclib.db_manager import DbManager @@ -14,6 +15,8 @@ class IDispatcher(ABC): """Dispatcher base class with core functionality for manage cache and background process""" + + container:'containers.Container' @property @abstractmethod diff --git a/bclib/dispatcher/named_pipe_dispatcher.py b/bclib/dispatcher/named_pipe_dispatcher.py index a53e6af..91da0f2 100644 --- a/bclib/dispatcher/named_pipe_dispatcher.py +++ b/bclib/dispatcher/named_pipe_dispatcher.py @@ -1,6 +1,6 @@ import asyncio from sys import platform -from ..dispatcher.routing_dispatcher import RoutingDispatcher +from .routing_dispatcher import RoutingDispatcher from bclib.listener import Message diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index e48a9b5..4f45f04 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -4,20 +4,22 @@ import re from struct import error from typing import Callable, Any, Coroutine, Optional +from dependency_injector import containers +from bclib.logger import ILogger +from bclib.db_manager import DbManager +from bclib.cache import CacheManager from bclib.utility import DictEx - from bclib.context import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext, NamedPipeContext from bclib.listener import Message, MessageType, HttpBaseDataType, ReceiveMessage - -from bclib.dispatcher.dispatcher_helper import DispatcherHelper -from bclib.dispatcher.dispatcher import Dispatcher +from .dispatcher_helper import DispatcherHelper +from .dispatcher import Dispatcher class RoutingDispatcher(Dispatcher, DispatcherHelper): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) self.__default_router = self.options.defaultRouter\ if 'defaultRouter' in self.options and isinstance(self.options.defaultRouter, str)\ else None diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index 7227ef5..84befcc 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -1,11 +1,17 @@ import asyncio -from ..dispatcher.routing_dispatcher import RoutingDispatcher -from ..listener import Endpoint, Message, SocketListener +from dependency_injector import containers + +from bclib.cache import CacheManager +from bclib.db_manager import DbManager +from bclib.logger import ILogger +from bclib.utility import DictEx +from bclib.listener import Endpoint, Message, SocketListener +from .routing_dispatcher import RoutingDispatcher class SocketDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) + def __init__(self, container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) self.__lock = asyncio.Lock() self.__listener = SocketListener( Endpoint(self.options.receiver), diff --git a/bclib/edge.py b/bclib/edge.py index 46285bd..cb47eaa 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -1,6 +1,8 @@ """Main module of bclib.wrapper for all exist module that need in basic coding""" import asyncio +import collections +from dependency_injector import containers,providers from bclib.db_manager import * from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, NamedPipeDispatcher, EndpointDispatcher from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, NamedPipeContext @@ -8,9 +10,9 @@ from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate from bclib.exception import * +from bclib.edge_container import EdgeContainer from bclib import __version__ - def from_config(option_file_path: str, file_name: str = "host.json"): """Create related RoutingDispatcher obj from config file in related path""" import json @@ -75,8 +77,38 @@ def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> Routing ret_val = SocketDispatcher(options=options,loop=loop) return ret_val +def __get_arg_parts(container:'EdgeContainer'): + import sys + import getopt + + argument_list = sys.argv[1:] + # Options + short_options = "mn:" + # Long options + long_options = ["Name =", "Multi"] + is_multi = False + try: + arguments, _ = getopt.gnu_getopt( + argument_list, short_options, long_options) + for current_argument, current_value in arguments: + if current_argument in ("-n", "--Name"): + container.app_config.name.from_value(current_value.strip()) + elif current_argument in ("-m", "--Multi"): + is_multi = True + container.app_config.is_multi.from_value(is_multi) + except getopt.error as err: + print(str(err)) + +def create_server(container:'EdgeContainer') -> RoutingDispatcher: + """Create related RoutingDispatcher obj from config object""" -def __print_splash(inMultiMode: bool): + container.app_container.override(providers.Object(container)) + __get_arg_parts(container) + if not container.app_config.is_multi(): + __print_splash(False) + return container.dispatcher() + +def __print_splash(in_multi_mode: bool): print(f''' ______ _ _____ _ | ___ \\ (_) | ___| | | @@ -92,7 +124,7 @@ def __print_splash(inMultiMode: bool): Welcome To BasisCore Ecosystem Follow us on https://BasisCore.com/ bclib Version : {__version__} -Run in {'multi' if inMultiMode else 'single'} instance mode! +Run in {'multi' if in_multi_mode else 'single'} instance mode! *********************************** (Press CTRL+C to quit) ''') diff --git a/bclib/edge_container.py b/bclib/edge_container.py new file mode 100644 index 0000000..0560785 --- /dev/null +++ b/bclib/edge_container.py @@ -0,0 +1,50 @@ +import asyncio +import sys + +from dependency_injector import containers,providers +from bclib.logger import LoggerFactory +from bclib.db_manager import DbManager +from bclib.cache import CacheFactory +from bclib.utility import DictEx +from bclib.dispatcher import SocketDispatcher, DevServerDispatcher, EndpointDispatcher + +def get_mode(options:'DictEx'): + if options.has("server"): + ret_val = 'server' + elif options.has("endpoint"): + ret_val = 'endpoint' + else: + ret_val = 'socket' + return ret_val + +def create_loop(options:'DictEx') -> asyncio.AbstractEventLoop: + loop:asyncio.AbstractEventLoop = options.loop + if loop is None and sys.platform == 'win32': + # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. + # Use Windows version of proactor event loop using IOCP + loop = asyncio.ProactorEventLoop() + current_loop = asyncio.get_event_loop() + if loop is not None and current_loop != loop: + asyncio.set_event_loop(loop) + return asyncio.get_event_loop() + +class EdgeContainer(containers.DeclarativeContainer): + app_config = providers.Configuration() + app_container = providers.Object(providers.Self()) + app_options = providers.Singleton( DictEx,app_config) + app_mode = providers.Singleton( get_mode,app_options) + app_cache_options =providers.Singleton(lambda x:x.cache,app_options) + app_event_loop = providers.Singleton(create_loop,app_options) + app_cache_manager = providers.Singleton(CacheFactory.create,app_cache_options) + app_db_manager = providers.Singleton(DbManager,app_options, app_event_loop) + app_logger= providers.Singleton(LoggerFactory.create,app_options) + app_server_dispatcher = providers.Singleton(DevServerDispatcher,app_container, app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_endpoint_dispatcher = providers.Singleton(EndpointDispatcher,app_container,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_socket_dispatcher = providers.Singleton(SocketDispatcher,app_container,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + + dispatcher = providers.Selector( + app_mode, + server=app_server_dispatcher, + endpoint = app_endpoint_dispatcher, + socket = app_socket_dispatcher + ) \ No newline at end of file diff --git a/bclib/utility/dict_ex.py b/bclib/utility/dict_ex.py index e971176..a984b5d 100644 --- a/bclib/utility/dict_ex.py +++ b/bclib/utility/dict_ex.py @@ -4,7 +4,7 @@ class DictEx(dict): """Extended version of dict class for dot.notation access to attributes""" - def __init__(self, *args, **kwargs): + def __init__(self, *args): super().__init__() for arg in args: if isinstance(arg, dict): diff --git a/test/dev_server/simple.py b/test/dev_server/simple.py index f18decc..4719515 100644 --- a/test/dev_server/simple.py +++ b/test/dev_server/simple.py @@ -1,27 +1,52 @@ +import asyncio +import random from bclib import edge - -if "options" not in dir(): - options = { +from bclib.edge import EdgeContainer +from dependency_injector import containers, providers +from dependency_injector.wiring import Provide, inject + +def manage_resource1(): + print("init tasks 1") + yield + print("stop tasks 1") +async def manage_resource2_async(): + await asyncio.sleep(.5) + print("init tasks 2") + yield None + await asyncio.sleep(.5) + print("stop tasks 2") + +@containers.copy(EdgeContainer) +class AppContainer(EdgeContainer): + app_resource1 = providers.Resource(manage_resource1) + app_resource2 = providers.Resource(manage_resource2_async) + object_provider = providers.Callable(lambda: random.randint(100,999)) + object_val = providers.Callable(lambda: random.randint(100,999)) + object_val2 = providers.Callable(lambda: random.randint(1000,9999)) + +container=AppContainer() +container.app_config.from_dict({ "server": "localhost:8080", - "router": "web" - } - + "router": "web", + #"loop":asyncio.get_event_loop() + "cache":{"1":22} + }) -app = edge.from_options(options) +app = edge.create_server(container) async def check_async(context: edge.RequestContext): return context.url.endswith("app") - @app.web_action(app.callback(check_async)) def process_web_action(context: edge.WebContext): return "result from process_web_action" - @app.web_action() -def process_default_web_action(context: edge.WebContext): - return "result from process_default_web_action" +@inject +def process_default_web_action(context: edge.WebContext,val=Provide["object_provider"],container:AppContainer =Provide["edge_container"]): + return "result from process_default_web_action " + str(val or "?") +" "+ str(context.container.object_val()) + " "+ str(container.object_val2()) +container.wire(modules=[__name__]) app.listening() From 5102b3d5a1daef45347220822419218db66e7a54 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:34:16 +0330 Subject: [PATCH 03/15] remove named pip dispatcher --- bclib/context/context.py | 4 - bclib/db_manager/__init__.py | 3 - bclib/db_manager/db_manager.py | 8 -- bclib/db_manager/named_pipe/__init__.py | 3 - .../named_pipe/inamed_pipe_connection.py | 13 -- .../named_pipe/linux_named_pipe_connection.py | 78 ------------ .../named_pipe/named_pipe_connection.py | 30 ----- .../windows_named_pipe_connection.py | 104 --------------- bclib/dispatcher/__init__.py | 1 - bclib/dispatcher/dispatcher.py | 1 + bclib/dispatcher/named_pipe_dispatcher.py | 35 ------ bclib/edge.py | 13 +- bclib/listener/__init__.py | 2 - bclib/listener/linux_named_pipe_listener.py | 113 ----------------- bclib/listener/windows_named_pipe_listener.py | 109 ---------------- bclib/utility/__init__.py | 2 - bclib/utility/linux_named_pipe_helper.py | 64 ---------- bclib/utility/windows_named_pipe_helper.py | 119 ------------------ test/all_test.py | 2 +- test/dev_server/simple.py | 2 +- 20 files changed, 8 insertions(+), 698 deletions(-) delete mode 100644 bclib/db_manager/named_pipe/__init__.py delete mode 100644 bclib/db_manager/named_pipe/inamed_pipe_connection.py delete mode 100644 bclib/db_manager/named_pipe/linux_named_pipe_connection.py delete mode 100644 bclib/db_manager/named_pipe/named_pipe_connection.py delete mode 100644 bclib/db_manager/named_pipe/windows_named_pipe_connection.py delete mode 100644 bclib/dispatcher/named_pipe_dispatcher.py delete mode 100644 bclib/listener/linux_named_pipe_listener.py delete mode 100644 bclib/listener/windows_named_pipe_listener.py delete mode 100644 bclib/utility/linux_named_pipe_helper.py delete mode 100644 bclib/utility/windows_named_pipe_helper.py diff --git a/bclib/context/context.py b/bclib/context/context.py index dfac6c7..d91036d 100644 --- a/bclib/context/context.py +++ b/bclib/context/context.py @@ -23,10 +23,6 @@ def __init__(self, dispatcher: 'dispatcher.IDispatcher') -> None: self.url: Optional[str] = None self.is_adhoc = True - @property - def container(self): - return self.dispatcher.container - def open_sql_connection(self, key: str) -> SqlDb: return self.dispatcher.db_manager.open_sql_connection(key) diff --git a/bclib/db_manager/__init__.py b/bclib/db_manager/__init__.py index ff7fd85..de012b3 100644 --- a/bclib/db_manager/__init__.py +++ b/bclib/db_manager/__init__.py @@ -4,6 +4,3 @@ from bclib.db_manager.mongo_db import MongoDb from bclib.db_manager.rabbit_connection import RabbitConnection from bclib.db_manager.restful_connection import RESTfulConnection -from bclib.db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from bclib.db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection -from bclib.db_manager.named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection diff --git a/bclib/db_manager/db_manager.py b/bclib/db_manager/db_manager.py index a5b498f..75abfa0 100644 --- a/bclib/db_manager/db_manager.py +++ b/bclib/db_manager/db_manager.py @@ -1,7 +1,5 @@ import asyncio from bclib.utility import DictEx -from ..db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from ..db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection from ..db_manager.rabbit_connection import RabbitConnection from ..db_manager.db import Db from ..db_manager.mongo_db import MongoDb @@ -42,9 +40,6 @@ def open_connection(self, key: str) -> Db: ret_val = RESTfulConnection(setting) elif db_type == "rabbit": ret_val = RabbitConnection(setting) - elif db_type == "named_pipe": - ret_val = NamedPipeConnection.get_connection( - setting, self._event_loop) else: print( f"Data base of type '{db_type}' not supported in this version") @@ -64,6 +59,3 @@ def open_restful_connection(self, key: str) -> RESTfulConnection: def open_rabbit_connection(self, key: str) -> RabbitConnection: return self.open_connection(key) - - def get_named_pipe_connection(self, key: str) -> INamedPipeConnection: - return self.open_connection(key) diff --git a/bclib/db_manager/named_pipe/__init__.py b/bclib/db_manager/named_pipe/__init__.py deleted file mode 100644 index 5e9e312..0000000 --- a/bclib/db_manager/named_pipe/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bclib.db_manager.named_pipe.named_pipe_connection import NamedPipeConnection -from bclib.db_manager.named_pipe.inamed_pipe_connection import INamedPipeConnection -from bclib.db_manager.named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection diff --git a/bclib/db_manager/named_pipe/inamed_pipe_connection.py b/bclib/db_manager/named_pipe/inamed_pipe_connection.py deleted file mode 100644 index bf516fb..0000000 --- a/bclib/db_manager/named_pipe/inamed_pipe_connection.py +++ /dev/null @@ -1,13 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class INamedPipeConnection(ABC): - - @abstractmethod - def try_send_command(self, command: 'Any') -> bool: - pass - - @abstractmethod - async def try_send_query_async(self, query: 'Any') -> 'Any': - pass diff --git a/bclib/db_manager/named_pipe/linux_named_pipe_connection.py b/bclib/db_manager/named_pipe/linux_named_pipe_connection.py deleted file mode 100644 index 57311a8..0000000 --- a/bclib/db_manager/named_pipe/linux_named_pipe_connection.py +++ /dev/null @@ -1,78 +0,0 @@ -import asyncio -from io import BufferedReader, BufferedWriter -import json -import uuid -from typing import Any -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - -from bclib.utility import LinuxNamedPipeHelper - - -class LinuxNamedPipeConnection(INamedPipeConnection): - def __init__(self, pipe_name: str, event_loop: asyncio.AbstractEventLoop) -> None: - self.__event_loop = event_loop - self.__reader_pipe_name = F"{pipe_name}-reader" - self.__writer_pipe_name = F"{pipe_name}-writer" - self.__reader_pipe_handle: BufferedWriter = None - self.__writer_pipe_handle: BufferedReader = None - self.__query_list: 'dict[str,asyncio.Future]' = dict() - self.__propcess_tesk = self.__event_loop.create_task( - self.__get_receive_message_Task()) - - async def __get_receive_message_Task(self): - while True: - try: - if self.__writer_pipe_handle is None: - self.__writer_pipe_handle = open( - self.__writer_pipe_name, 'rb') - message = await LinuxNamedPipeHelper.read_from_named_pipe_async(self.__writer_pipe_handle, self.__event_loop) - if message and message.session_id in self.__query_list: - future = self.__query_list[message.session_id] - del self.__query_list[message.session_id] - data = json.loads(message.buffer) - future.get_loop().call_soon_threadsafe(future.set_result, data) - except asyncio.CancelledError: - self.clear_query_list() - break - except: - self.__writer_pipe_handle = None - self.clear_query_list() - await asyncio.sleep(1) - - def __try_send_command(self, command: Any) -> str: - from bclib.listener import Message - session_id = None - try: - if self.__reader_pipe_handle is None: - self.__reader_pipe_handle = open(self.__reader_pipe_name, "wb") - session_id = str(uuid.uuid4()) - msg = Message.create_add_hock( - session_id, json.dumps(command).encode("utf-8")) - LinuxNamedPipeHelper.write_to_named_pipe( - msg, self.__reader_pipe_handle) - except: - self.__reader_pipe_handle = None - raise - return session_id - - def try_send_command(self, command: Any) -> bool: - return True if self.__try_send_command(command) else False - - async def try_send_query_async(self, query: 'Any') -> 'Any': - ret_val = None - session_id = self.__try_send_command(query) - if session_id: - future = self.__event_loop.create_future() - self.__query_list[session_id] = future - ret_val = await future - return ret_val - - def clear_query_list(self): - for session_id in self.__query_list: - try: - future = self.__query_list[session_id] - future.get_loop().call_soon_threadsafe( - future.set_exception, Exception("Named Pipe is broken")) - except: - pass - self.__query_list.clear() diff --git a/bclib/db_manager/named_pipe/named_pipe_connection.py b/bclib/db_manager/named_pipe/named_pipe_connection.py deleted file mode 100644 index 9e32d8b..0000000 --- a/bclib/db_manager/named_pipe/named_pipe_connection.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -from sys import platform -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - - -class NamedPipeConnection: - __dic: 'dict[str,INamedPipeConnection]' = dict() - - @staticmethod - def get_connection(pipe_name: str, loop: asyncio.AbstractEventLoop) -> INamedPipeConnection: - ret_val = None - if pipe_name in NamedPipeConnection.__dic: - ret_val = NamedPipeConnection.__dic[pipe_name] - else: - # https://docs.python.org/3/library/sys.html#sys.platform - if platform == "linux" or platform == "linux2": - # linux - from ..named_pipe.linux_named_pipe_connection import LinuxNamedPipeConnection - ret_val = NamedPipeConnection.__dic[pipe_name] = LinuxNamedPipeConnection( - pipe_name, loop) - elif platform == "darwin": - # OS X - raise Exception( - "named pipe connection not implemented in OS X") - elif platform == "win32": - from ..named_pipe.windows_named_pipe_connection import WindowsNamedPipeConnection - ret_val = NamedPipeConnection.__dic[pipe_name] = WindowsNamedPipeConnection( - pipe_name, loop) - NamedPipeConnection.__dic[pipe_name] = ret_val - return ret_val diff --git a/bclib/db_manager/named_pipe/windows_named_pipe_connection.py b/bclib/db_manager/named_pipe/windows_named_pipe_connection.py deleted file mode 100644 index d54d439..0000000 --- a/bclib/db_manager/named_pipe/windows_named_pipe_connection.py +++ /dev/null @@ -1,104 +0,0 @@ -import asyncio -import json -import uuid -from typing import Any -from ..named_pipe.inamed_pipe_connection import INamedPipeConnection - -from bclib.utility import WindowsNamedPipeHelper - - -class WindowsNamedPipeConnection(INamedPipeConnection): - def __init__(self, pipe_name: str, event_loop: asyncio.AbstractEventLoop) -> None: - self.__event_loop = event_loop - self.__reader_pipe_name = F"{pipe_name}/reader" - self.__writer_pipe_name = F"{pipe_name}/writer" - self.__reader_pipe_handle = None - self.__writer_pipe_handle = None - self.__query_list: 'dict[str,asyncio.Future]' = dict() - self.__propcess_tesk = self.__event_loop.create_task( - self.__get_receive_message_Task()) - - def __connect_to_reader_named_pipe(self): - import win32file - import pywintypes - try: - self.__reader_pipe_handle = win32file.CreateFile(self.__reader_pipe_name, win32file.GENERIC_WRITE, - 0, None, win32file.OPEN_EXISTING, win32file.FILE_ATTRIBUTE_NORMAL, None) - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe '{self.__reader_pipe_name}'. {repr(e)}") - else: - print( - f"Named Pipe '{self.__reader_pipe_name}' error code {e.args[0]}. {repr(e)}") - raise - - def __connect_to_writer_pipe(self): - import win32file - import pywintypes - try: - self.__writer_pipe_handle = win32file.CreateFile(self.__writer_pipe_name, win32file.GENERIC_READ, - 0, None, win32file.OPEN_EXISTING, win32file.FILE_ATTRIBUTE_READONLY, None) - - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe '{self.__writer_pipe_name}'. {repr(e)}") - else: - print( - f"Named Pipe '{self.__writer_pipe_name}' error code {e.args[0]}. {repr(e)}") - raise - - async def __get_receive_message_Task(self): - while True: - try: - if self.__writer_pipe_handle is None: - self.__connect_to_writer_pipe() - message = await WindowsNamedPipeHelper.read_from_named_pipe_async(self.__writer_pipe_handle, self.__event_loop) - if message and message.session_id in self.__query_list: - future = self.__query_list[message.session_id] - del self.__query_list[message.session_id] - data = json.loads(message.buffer) - future.get_loop().call_soon_threadsafe(future.set_result, data) - except asyncio.CancelledError: - self.clear_query_list() - break - except: - self.__writer_pipe_handle = None - self.clear_query_list() - await asyncio.sleep(1) - - def __try_send_command(self, command: Any) -> str: - from bclib.listener import Message - session_id = None - try: - if self.__reader_pipe_handle is None: - self.__connect_to_reader_named_pipe() - session_id = str(uuid.uuid4()) - msg = Message.create_add_hock( - session_id, json.dumps(command).encode("utf-8")) - WindowsNamedPipeHelper.write_to_named_pipe( - msg, self.__reader_pipe_handle) - except: - self.__reader_pipe_handle = None - return session_id - - def try_send_command(self, command: Any) -> bool: - return True if self.__try_send_command(command) else False - - async def try_send_query_async(self, query: 'Any') -> 'Any': - ret_val = None - session_id = self.__try_send_command(query) - if session_id: - future = self.__event_loop.create_future() - self.__query_list[session_id] = future - ret_val = await future - return ret_val - - def clear_query_list(self): - for session_id in self.__query_list: - try: - future = self.__query_list[session_id] - future.get_loop().call_soon_threadsafe( - future.set_exception, Exception("Named Pipe is broken")) - except: - pass - self.__query_list.clear() diff --git a/bclib/dispatcher/__init__.py b/bclib/dispatcher/__init__.py index 2f7b440..2674718 100644 --- a/bclib/dispatcher/__init__.py +++ b/bclib/dispatcher/__init__.py @@ -3,6 +3,5 @@ from bclib.dispatcher.socket_dispatcher import SocketDispatcher from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher from bclib.dispatcher.routing_dispatcher import RoutingDispatcher -from bclib.dispatcher.named_pipe_dispatcher import NamedPipeDispatcher from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher from bclib.dispatcher.dispatcher_helper import DispatcherHelper diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index 71bbbf5..579adb6 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -383,6 +383,7 @@ def initialize_task(self): for dispatcher in self.__rabbit_dispatcher: dispatcher.initialize_task(self.event_loop) + #TODO:pre ansd post callback replaced with resource provider of DI def listening(self, with_block:bool = True): """Start listening to request for process""" for sig in (signal.SIGTERM, signal.SIGINT): diff --git a/bclib/dispatcher/named_pipe_dispatcher.py b/bclib/dispatcher/named_pipe_dispatcher.py deleted file mode 100644 index 91da0f2..0000000 --- a/bclib/dispatcher/named_pipe_dispatcher.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio -from sys import platform -from .routing_dispatcher import RoutingDispatcher -from bclib.listener import Message - - -class NamedPipeDispatcher(RoutingDispatcher): - def __init__(self, options: dict,loop:asyncio.AbstractEventLoop=None): - super().__init__(options=options,loop=loop) - self.__lock = asyncio.Lock() - # https://docs.python.org/3/library/sys.html#sys.platform - if platform == "linux" or platform == "linux2": - # linux - from bclib.listener import LinuxNamedPipeListener - self.__listener = LinuxNamedPipeListener( - self.options.named_pipe, - self._on_message_receive_async) - elif platform == "win32": - from bclib.listener import WindowsNamedPipeListener - self.__listener = WindowsNamedPipeListener( - self.options.named_pipe, - self._on_message_receive_async) - elif platform == "darwin": - # OS X - raise Exception("named pipe not implemented in OS X") - - async def send_message_async(self, message: Message) -> bool: - """Send message to endpoint""" - - async with self.__lock: - return await self.__listener.send_message_async(message) - - def initialize_task(self): - super().initialize_task() - self.__listener.initialize_task(self.event_loop) diff --git a/bclib/edge.py b/bclib/edge.py index cb47eaa..053df95 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -1,12 +1,11 @@ """Main module of bclib.wrapper for all exist module that need in basic coding""" import asyncio -import collections -from dependency_injector import containers,providers +from dependency_injector import providers from bclib.db_manager import * -from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, NamedPipeDispatcher, EndpointDispatcher +from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, NamedPipeContext -from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders, WindowsNamedPipeHelper +from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate from bclib.exception import * @@ -48,14 +47,14 @@ def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> Routing import getopt multi: bool = False - argumentList = sys.argv[1:] + argument_list = sys.argv[1:] # Options short_options = "mn:" # Long options long_options = ["Name =", "Multi"] try: arguments, _ = getopt.gnu_getopt( - argumentList, short_options, long_options) + argument_list, short_options, long_options) for current_argument, current_value in arguments: if current_argument in ("-n", "--Name"): options["name"] = current_value.strip() @@ -69,8 +68,6 @@ def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> Routing ret_val: RoutingDispatcher = None if "server" in options: ret_val = DevServerDispatcher(options=options,loop=loop) - elif "named_pipe" in options: - ret_val = NamedPipeDispatcher(options=options,loop=loop) elif "endpoint" in options: ret_val = EndpointDispatcher(options=options,loop=loop) else: diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index aa15891..ebf9e3f 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -7,6 +7,4 @@ from bclib.listener.http_listener.http_listener import HttpListener from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType -from bclib.listener.windows_named_pipe_listener import WindowsNamedPipeListener -from bclib.listener.linux_named_pipe_listener import LinuxNamedPipeListener from bclib.listener.web_message import WebMessage diff --git a/bclib/listener/linux_named_pipe_listener.py b/bclib/listener/linux_named_pipe_listener.py deleted file mode 100644 index f824d14..0000000 --- a/bclib/listener/linux_named_pipe_listener.py +++ /dev/null @@ -1,113 +0,0 @@ -import asyncio -from io import BufferedWriter -import os -import sys -from typing import Callable, Coroutine -from ..listener.message import Message -from bclib.utility import LinuxNamedPipeHelper - - -class LinuxNamedPipeListener: - """""" - - def __init__(self, pipe_name: str, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): - self.pipe_name = pipe_name - self.__writer_pipe_name = F"{self.pipe_name}-writer" - self.on_message_receive = on_message_receive_call_back - self.__writer_pipe: BufferedWriter = None - self.__event_loop: asyncio.AbstractEventLoop = None - - def try_unlink(self, pipe_name: str): - try: - os.unlink(pipe_name) - print(f"unlink named pipe '{pipe_name}'...") - except: - pass - - async def __connect_writer_pipe_async(self): - if self.__writer_pipe: - self.try_unlink(self.__writer_pipe_name) - - try: - os.mkfifo(self.__writer_pipe_name) - except Exception as ex: - print('error in run os.mkfifo():', ex) - - try: - self.__writer_pipe = open(self.__writer_pipe_name, "wb") - print( - f"Writer named pipe '{self.__writer_pipe_name}' is created...") - except Exception as ex: - print(f"Error in create writer named pipe. {repr(ex)}") - self.try_unlink(self.__writer_pipe_name) - raise - - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - - async def send_message_async(self, message: Message) -> bool: - try_count = 0 - send = False - while not send: - try: - if self.__writer_pipe is None: - await self.__connect_writer_pipe_async() - LinuxNamedPipeHelper.write_to_named_pipe( - message, self.__writer_pipe) - send = True - except asyncio.CancelledError: - break - except Exception as ex: - try_count = try_count+1 - if self.__writer_pipe: - self.try_unlink(self.__writer_pipe_name) - try: - self.__writer_pipe.close() - except: - pass - self.__writer_pipe = None - print(f"Error in send message {ex}") - if try_count > 3: - break - await asyncio.sleep(.5) - return send - - def initialize_task(self, loop: asyncio.AbstractEventLoop): - self.__event_loop = loop - - async def reader_loop_async(): - reader_pipe_name = F"{self.pipe_name}-reader" - while True: - try: - os.mkfifo(reader_pipe_name) - except Exception as ex: - print( - f'Error in call os.mkfifo() for ${reader_pipe_name}', ex) - - with open(reader_pipe_name, "rb") as reader_pipe: - print( - f"Reader named pipe '{reader_pipe_name}' is created. try to read from it...") - try: - while True: - message = await LinuxNamedPipeHelper.read_from_named_pipe_async(reader_pipe, self.__event_loop) - if message: - self.__event_loop.create_task( - self.__process_message_async(message)) - except asyncio.CancelledError: - print('Edge named pipe server stopped.!') - break - except Exception as ex: - print('Error cause edge named pipe server restart!', ex) - await asyncio.sleep(1) - finally: - try: - if reader_pipe: - os.unlink(reader_pipe_name) - print("unlink...") - except Exception as ex: - print("Error in unlink named pipe...", ex) - - self.__event_loop.create_task(reader_loop_async()) - self.__event_loop.create_task(self.__connect_writer_pipe_async()) diff --git a/bclib/listener/windows_named_pipe_listener.py b/bclib/listener/windows_named_pipe_listener.py deleted file mode 100644 index c56f1b2..0000000 --- a/bclib/listener/windows_named_pipe_listener.py +++ /dev/null @@ -1,109 +0,0 @@ -import asyncio -from typing import Callable, Coroutine -from ..listener.message import Message -from bclib.utility import WindowsNamedPipeHelper - - -class WindowsNamedPipeListener: - """""" - - def __init__(self, pipe_name: str, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): - self.pipe_name = pipe_name - self.on_message_receive = on_message_receive_call_back - self.__writer_pipe = None - self.__reader_pipe = None - self.__event_loop: asyncio.AbstractEventLoop = None - - async def __connect_writer_pipe_async(self): - import win32pipe - import pywintypes - try: - name = F"{self.pipe_name}/writer" - self.__writer_pipe = win32pipe.CreateNamedPipe(name, win32pipe.PIPE_ACCESS_OUTBOUND, - win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, - 1, 65536, 65536, 0, None) - print( - f"Writer named pipe '{name}' is created. Waiting for reader client to connect...") - await WindowsNamedPipeHelper.wait_for_client_connect_async(self.__writer_pipe, self.__event_loop) - print( - f"Reader client is connected to '{name}'...") - except pywintypes.error as e: # pylint: disable=maybe-no-member - self.__writer_pipe = None - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe. {repr(e)}") - else: - print(f"Named Pipe error code {e.args[0]}. {repr(e)}") - raise - except Exception as ex: - self.__writer_pipe = None - print(f"Error in create writer named pipe. {repr(ex)}") - raise - - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - - async def send_message_async(self, message: Message) -> bool: - try_count = 0 - send = False - while not send: - try: - if self.__writer_pipe is None: - await self.__connect_writer_pipe_async() - WindowsNamedPipeHelper.write_to_named_pipe( - message, self.__writer_pipe) - send = True - except asyncio.CancelledError: - break - except Exception as ex: - try_count = try_count+1 - self.__writer_pipe = None - print(f"Error in send message {ex}") - if try_count > 3: - break - await asyncio.sleep(.5) - return send - - def initialize_task(self, loop: asyncio.AbstractEventLoop): - self.__event_loop = loop - - async def reader_loop_async(): - import win32pipe - import win32file - import pywintypes - while True: - try: - name = F"{self.pipe_name}/reader" - self.__reader_pipe = win32pipe.CreateNamedPipe(name, win32pipe.PIPE_ACCESS_INBOUND, - win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT, - 1, 65536, 65536, 0, None) - print( - f"Reader named pipe '{name}' is created. Waiting for writer client to connect...") - await WindowsNamedPipeHelper.wait_for_client_connect_async(self.__reader_pipe, self.__event_loop) - print( - f"Writer client connect to '{name}'...") - while True: - message = await WindowsNamedPipeHelper.read_from_named_pipe_async( - self.__reader_pipe, self.__event_loop) - if message: - self.__event_loop.create_task( - self.__process_message_async(message)) - except asyncio.CancelledError: - print('Edge named pipe server stopped.!') - break - except pywintypes.error as e: # pylint: disable=maybe-no-member - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No reader named pipe. {repr(e)}") - elif e.args[0] == 109: # ERROR_BROKEN_PIPE - print(f"Reader named pipe is broken. {repr(e)}") - else: - print( - f"Reader named pipe error code {e.args[0]}. {repr(e)}") - finally: - # Disconnect the named pipe - win32pipe.DisconnectNamedPipe(self.__reader_pipe) - # CLose the named pipe - win32file.CloseHandle(self.__reader_pipe) - self.__event_loop.create_task(reader_loop_async()) - self.__event_loop.create_task(self.__connect_writer_pipe_async()) diff --git a/bclib/utility/__init__.py b/bclib/utility/__init__.py index a689ee5..1e80c42 100644 --- a/bclib/utility/__init__.py +++ b/bclib/utility/__init__.py @@ -3,5 +3,3 @@ from bclib.utility.http_mime_types import HttpMimeTypes from bclib.utility.response_types import ResponseTypes from bclib.utility.http_headers import HttpHeaders -from bclib.utility.windows_named_pipe_helper import WindowsNamedPipeHelper -from bclib.utility.linux_named_pipe_helper import LinuxNamedPipeHelper diff --git a/bclib/utility/linux_named_pipe_helper.py b/bclib/utility/linux_named_pipe_helper.py deleted file mode 100644 index 768a71c..0000000 --- a/bclib/utility/linux_named_pipe_helper.py +++ /dev/null @@ -1,64 +0,0 @@ -import asyncio -from io import BufferedReader, BufferedWriter -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from bclib.listener.message import Message - - -class LinuxNamedPipeHelper: - - @staticmethod - def __read_from_named_pipe(named_pipe_handle: BufferedReader, future: asyncio.Future) -> None: - from bclib.listener import Message, MessageType - - try: - data = named_pipe_handle.read(1) - if data: - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = named_pipe_handle.read(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = named_pipe_handle.read(data_len) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = named_pipe_handle.read(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - parameter = named_pipe_handle.read( - data_len) - message = Message(session_id, message_type, parameter) - else: - message = None - future.get_loop().call_soon_threadsafe(future.set_result, message) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - - @staticmethod - async def read_from_named_pipe_async(named_pipe_handle: BufferedReader, loop: asyncio.AbstractEventLoop = None) -> 'Message': - if not loop: - loop = asyncio.get_event_loop() - future = loop.create_future() - loop.run_in_executor( - None, LinuxNamedPipeHelper.__read_from_named_pipe, named_pipe_handle, future) - return await future - - @staticmethod - def write_to_named_pipe(message: 'Message', named_pipe_handle: BufferedWriter): - from bclib.listener import MessageType - - named_pipe_handle.write(message.type.value.to_bytes(1, 'big')) - - data = message.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - named_pipe_handle.write(data_length_bytes) - named_pipe_handle.write(data) - - if message.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(message.buffer).to_bytes(4, 'big') - named_pipe_handle.write(data_length_bytes) - named_pipe_handle.write(message.buffer) - - named_pipe_handle.flush() diff --git a/bclib/utility/windows_named_pipe_helper.py b/bclib/utility/windows_named_pipe_helper.py deleted file mode 100644 index 4dce194..0000000 --- a/bclib/utility/windows_named_pipe_helper.py +++ /dev/null @@ -1,119 +0,0 @@ -import asyncio -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from bclib.listener.message import Message - - -class WindowsNamedPipeHelper: - - @staticmethod - def __check_read_error(error_code): - if error_code != 0: - raise Exception( - f"error in read from pip. error code = {error_code}") - - @staticmethod - def __check_write_error(error_code): - if error_code != 0: - raise Exception( - f"error in write in pip. error code = {error_code}") - - @staticmethod - def __read_from_named_pipe(named_pipe_handle: Any, future: asyncio.Future) -> None: - import win32file - from bclib.listener import Message, MessageType - - try: - error, data = win32file.ReadFile(named_pipe_handle, 1) - WindowsNamedPipeHelper.__check_read_error(error) - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - error, data = win32file.ReadFile(named_pipe_handle, 4) - WindowsNamedPipeHelper.__check_read_error(error) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - error, data = win32file.ReadFile(named_pipe_handle, data_len) - WindowsNamedPipeHelper.__check_read_error(error) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - error, data = win32file.ReadFile(named_pipe_handle, 4) - WindowsNamedPipeHelper.__check_read_error(error) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - error, parameter = win32file.ReadFile( - named_pipe_handle, data_len) - WindowsNamedPipeHelper.__check_read_error(error) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - else: - message = Message(session_id, message_type, parameter) - future.get_loop().call_soon_threadsafe(future.set_result, message) - - @staticmethod - async def read_from_named_pipe_async(named_pipe_handle: Any, loop: asyncio.AbstractEventLoop = None) -> 'Message': - if not loop: - loop = asyncio.get_event_loop() - future = loop.create_future() - loop.run_in_executor( - None, WindowsNamedPipeHelper.__read_from_named_pipe, named_pipe_handle, future) - return await future - - @staticmethod - def write_to_named_pipe(message: 'Message', named_pipe_handle: Any): - import win32file - import win32pipe - import pywintypes - from bclib.listener import MessageType - - try: - error, _ = win32file.WriteFile( - named_pipe_handle, message.type.value.to_bytes(1, 'big')) - WindowsNamedPipeHelper.__check_write_error(error) - data = message.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - error, _ = win32file.WriteFile( - named_pipe_handle, data_length_bytes) - WindowsNamedPipeHelper.__check_write_error(error) - error, _ = win32file.WriteFile(named_pipe_handle, data) - WindowsNamedPipeHelper.__check_write_error(error) - if message.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(message.buffer).to_bytes(4, 'big') - error, _ = win32file.WriteFile( - named_pipe_handle, data_length_bytes) - WindowsNamedPipeHelper.__check_write_error(error) - error, _ = win32file.WriteFile( - named_pipe_handle, message.buffer) - WindowsNamedPipeHelper.__check_write_error(error) - win32file.FlushFileBuffers(named_pipe_handle) - except pywintypes.error as e: # pylint: disable=maybe-no-member - # Disconnect the named pipe - win32pipe.DisconnectNamedPipe(named_pipe_handle) - # CLose the named pipe - win32file.CloseHandle(named_pipe_handle) - if e.args[0] == 2: # ERROR_FILE_NOT_FOUND - print(f"No Named Pipe. {repr(e)}") - elif e.args[0] == 232: - print(f"The Pipe is being closed. {repr(e)}") - elif e.args[0] == 109: # ERROR_BROKEN_PIPE - print(f"Named Pipe is broken. {repr(e)}") - else: - print(f"Named Pipe error code {e.args[0]}. {repr(e)}") - raise - - @staticmethod - async def wait_for_client_connect_async(pipe_handler: any, loop: asyncio.AbstractEventLoop = None) -> None: - import win32pipe - future = loop.create_future() - - def process(pipe_handler: any, future: asyncio.Future): - try: - win32pipe.ConnectNamedPipe(pipe_handler, None) - future.get_loop().call_soon_threadsafe(future.set_result, True) - except Exception as ex: - future.get_loop().call_soon_threadsafe(future.set_exception, ex) - if not loop: - loop = asyncio.get_event_loop() - loop.run_in_executor(None, process, pipe_handler, future) - await future diff --git a/test/all_test.py b/test/all_test.py index f14906f..7f201b4 100644 --- a/test/all_test.py +++ b/test/all_test.py @@ -185,7 +185,7 @@ def process_not_exist_message(context: edge.SocketContext): ChatRoom.process_message(context.message, None, None) -@ app.socket_action() +@app.socket_action() def process_all_other_message(context: edge.SocketContext): print("process_all_other_message") ChatRoom.process_message(context.message, diff --git a/test/dev_server/simple.py b/test/dev_server/simple.py index 4719515..a1b0ad1 100644 --- a/test/dev_server/simple.py +++ b/test/dev_server/simple.py @@ -45,7 +45,7 @@ def process_web_action(context: edge.WebContext): @app.web_action() @inject def process_default_web_action(context: edge.WebContext,val=Provide["object_provider"],container:AppContainer =Provide["edge_container"]): - return "result from process_default_web_action " + str(val or "?") +" "+ str(context.container.object_val()) + " "+ str(container.object_val2()) + return "result from process_default_web_action " + str(val or "?") +" "+ str(context.dispatcher.container.object_val()) + " "+ str(container.object_val2()) container.wire(modules=[__name__]) From 4ee769d68c274f9eafed1632bc3a3ce58d4134f7 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:49:08 +0330 Subject: [PATCH 04/15] add logger and context factory --- bclib/context/__init__.py | 2 +- bclib/context/client_source_context.py | 8 +- bclib/context/context.py | 26 +--- bclib/context/context_factory.py | 135 ++++++++++++++++++ bclib/context/json_base_request_context.py | 5 +- bclib/context/named_pipe_context.py | 14 -- bclib/context/rabbit_context.py | 2 +- bclib/context/request_context.py | 4 +- bclib/context/web_context.py | 8 +- bclib/dispatcher/dev_server_dispatcher.py | 5 +- bclib/dispatcher/dispatcher.py | 75 ++++------ bclib/dispatcher/endpoint_dispatcher.py | 5 +- bclib/dispatcher/routing_dispatcher.py | 119 +-------------- bclib/dispatcher/socket_dispatcher.py | 5 +- bclib/edge.py | 2 +- bclib/edge_container.py | 8 +- bclib/listener/http_listener/http_listener.py | 5 +- bclib/listener/rabbit_bus_listener.py | 4 +- .../exchange_rabbit_schema_base_logger.py | 2 +- bclib/logger/ilogger.py | 25 +++- bclib/logger/logger_factory.py | 14 +- bclib/logger/no_logger.py | 8 +- bclib/logger/rabbit_schema_base_logger.py | 6 +- bclib/logger/restful_schema_base_logger.py | 10 +- bclib/logger/schema_base_logger.py | 13 +- test/dev_server/simple.py | 4 +- 26 files changed, 257 insertions(+), 257 deletions(-) create mode 100644 bclib/context/context_factory.py delete mode 100644 bclib/context/named_pipe_context.py diff --git a/bclib/context/__init__.py b/bclib/context/__init__.py index b9a8483..5f46f54 100644 --- a/bclib/context/__init__.py +++ b/bclib/context/__init__.py @@ -11,4 +11,4 @@ from bclib.context.server_source_member_context import ServerSourceMemberContext from bclib.context.source_context import SourceContext from bclib.context.source_member_context import SourceMemberContext -from bclib.context.named_pipe_context import NamedPipeContext +from bclib.context.context_factory import ContextFactory diff --git a/bclib/context/client_source_context.py b/bclib/context/client_source_context.py index 647f127..738cd07 100644 --- a/bclib/context/client_source_context.py +++ b/bclib/context/client_source_context.py @@ -2,17 +2,17 @@ from bclib.parser import HtmlParserEx from bclib.utility import DictEx -from bclib.context.json_base_request_context import JsonBaseRequestContext +from .json_base_request_context import JsonBaseRequestContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher import IDispatcher + from bclib.listener import WebMessage class ClientSourceContext(JsonBaseRequestContext): """Context for client dbSource request""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher',message_object: 'WebMessage') -> None: super().__init__(cms_object, dispatcher,message_object) parser = HtmlParserEx() self.raw_command = self.cms.form.command diff --git a/bclib/context/context.py b/bclib/context/context.py index d91036d..9f42cda 100644 --- a/bclib/context/context.py +++ b/bclib/context/context.py @@ -1,42 +1,27 @@ from abc import ABC import traceback +import base64 from typing import TYPE_CHECKING, Optional, Tuple from bclib.exception import ShortCircuitErr - -from bclib.db_manager import SqlDb, SQLiteDb, MongoDb, RabbitConnection, RESTfulConnection from bclib.utility import DictEx, HttpStatusCodes, HttpStatusCodes from bclib.listener.http_listener import HttpBaseDataName, HttpBaseDataType -import base64 if TYPE_CHECKING: - from bclib import dispatcher + from bclib.dispatcher import IDispatcher class Context(ABC): """Base class for dispatching""" - def __init__(self, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, dispatcher: 'IDispatcher') -> None: super().__init__() self.dispatcher = dispatcher self.url_segments: DictEx = None self.url: Optional[str] = None self.is_adhoc = True - def open_sql_connection(self, key: str) -> SqlDb: - return self.dispatcher.db_manager.open_sql_connection(key) - - def open_sqllite_connection(self, key: str) -> SQLiteDb: - return self.dispatcher.db_manager.open_sqllite_connection(key) - - def open_mongo_connection(self, key: str) -> MongoDb: - return self.dispatcher.db_manager.open_mongo_connection(key) - - def open_restful_connection(self, key: str) -> RESTfulConnection: - return self.dispatcher.db_manager.open_restful_connection(key) - - def open_rabbit_connection(self, key: str) -> RabbitConnection: - return self.dispatcher.db_manager.open_rabbit_connection(key) + #TODO:Removed wrapper open_X_connection methode def generate_error_response(self, exception: Exception) -> dict: """Generate error response from process result""" @@ -60,7 +45,8 @@ def _generate_error_object(self, exception: Exception) -> 'Tuple[dict, str]': "errorCode": error_code, "errorMessage": str(exception) } - if self.dispatcher.log_error: + #TODO:use logger service + if True:#self.dispatcher.log_error: error_object["error"] = traceback.format_exc() return (error_object, status_code) diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py new file mode 100644 index 0000000..2004dc2 --- /dev/null +++ b/bclib/context/context_factory.py @@ -0,0 +1,135 @@ +import json +import re +from struct import error +from typing import Optional,TYPE_CHECKING + + +if TYPE_CHECKING: + from dispatcher.idispatcher import IDispatcher + from bclib.utility import DictEx + from bclib.logger import ILogger +from . import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext + + + +# from .client_source_context import ClientSourceContext +# from .restful_context import RESTfulContext +# from .web_context import WebContext +# from .request_context import RequestContext +# from .context import Context +# from .socket_context import SocketContext +# from .server_source_context import ServerSourceContext +# from .named_pipe_context import NamedPipeContext +from bclib.listener.message import Message, MessageType +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + +class ContextFactory: + def __init__(self,options:'DictEx',logger:'ILogger'): + self.logger = logger + self.__default_router = options.defaultRouter\ + if 'defaultRouter' in options and isinstance(options.defaultRouter, str)\ + else None + if options.has('router'): + router = options.router + if isinstance(router, str): + self.__context_type_detector: 'Callable[[str],str]' = lambda _: router + elif isinstance(router, DictEx): + self.__init_router_lookup(options.router) + else: + raise error( + "Invalid value for 'router' property in host options! Use string or dict object only.") + elif self.__default_router: + self.__context_type_detector: 'Callable[[str],str]' = lambda _: self.__default_router + else: + raise error( + "Invalid routing config! Please at least set one of 'router' or 'defaultRouter' property in host options.") + + def __init_router_lookup(self,router:'DictEx'): + """create router lookup dictionary""" + + route_dict = dict() + for key, values in router.items(): + k = k.strip() + if key != 'rabbit': + if '*' in values: + route_dict['*'] = key + break + else: + for value in values: + if len(value.strip()) != 0 and value not in route_dict: + route_dict[value] = key + if len(route_dict) == 1 and '*' in route_dict and self.__default_router is None: + router = route_dict['*'] + self.__context_type_detector: 'Callable[[str],str]' = lambda _: router + else: + self.__context_type_lookup = route_dict.items() + self.__context_type_detector = self.__context_type_detect_from_lookup + + def __context_type_detect_from_lookup(self, url: str) -> str: + """Detect context type from url about lookup""" + + context_type: Optional[str] = None + if url: + try: + for pattern, lookup_context_type in self.__context_type_lookup: + if pattern == "*" or re.search(pattern, url): + context_type = lookup_context_type + break + except TypeError: + pass + except error as ex: + print("Error in detect context from routing options!", ex) + return context_type if context_type else self.__default_router + + + def create_context(self, message: 'Message', dispatcher:'IDispatcher') -> Context: + """Create context from message object""" + + print('qam',message) + ret_val: RequestContext = None + context_type = None + cms_object: Optional[dict] = None + url: Optional[str] = None + request_id: Optional[str] = None + method: Optional[str] = None + message_json: Optional[dict] = None + if message.buffer is not None: + message_json = json.loads(message.buffer) + cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None + if cms_object: + if 'request' in cms_object: + req = cms_object["request"] + else: + raise KeyError("request key not found in cms object") + if 'full-url' in req: + url = req["full-url"] + else: + raise KeyError("full-url key not found in request") + request_id = req['request-id'] if 'request-id' in req else 'none' + method = req['methode'] if 'methode' in req else 'none' + if message.type == MessageType.AD_HOC: + if url or self.__default_router is None: + context_type = self.__context_type_detector(url) + else: + context_type = self.__default_router + else: + context_type = "socket" + self.logger.log_request( + f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") + + if context_type == "client_source": + ret_val = ClientSourceContext(cms_object, dispatcher,message) + elif context_type == "restful": + ret_val = RESTfulContext(cms_object, dispatcher,message) + elif context_type == "server_source": + ret_val = ServerSourceContext(message_json, dispatcher) + elif context_type == "web": + ret_val = WebContext(cms_object, dispatcher,message) + elif context_type == "socket": + ret_val = SocketContext(cms_object, dispatcher, message, message_json) + elif context_type is None: + raise NameError(f"No context found for '{url}'") + else: + raise NameError( + f"Configured context type '{context_type}' not found for '{url}'") + return ret_val diff --git a/bclib/context/json_base_request_context.py b/bclib/context/json_base_request_context.py index 0b4f0a6..bbedd33 100644 --- a/bclib/context/json_base_request_context.py +++ b/bclib/context/json_base_request_context.py @@ -2,11 +2,10 @@ from typing import TYPE_CHECKING from bclib.utility import HttpMimeTypes -from bclib.context.web_context import WebContext +from .web_context import WebContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib import dispatcher,listener class JsonBaseRequestContext(WebContext): diff --git a/bclib/context/named_pipe_context.py b/bclib/context/named_pipe_context.py deleted file mode 100644 index c45a9aa..0000000 --- a/bclib/context/named_pipe_context.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import TYPE_CHECKING -from . import Context -if TYPE_CHECKING: - from .. import dispatcher -from bclib.utility import DictEx - - -class NamedPipeContext(Context): - """Context for named pipe request""" - - def __init__(self, named_pipe_message: 'dict', raw_message: str, dispatcher: 'dispatcher.IDispatcher'): - super().__init__(dispatcher) - self.message: DictEx = DictEx(named_pipe_message) - self.raw_message = raw_message diff --git a/bclib/context/rabbit_context.py b/bclib/context/rabbit_context.py index 9f4049e..39edb97 100644 --- a/bclib/context/rabbit_context.py +++ b/bclib/context/rabbit_context.py @@ -1,6 +1,6 @@ import json from typing import Any, TYPE_CHECKING -from ..context.context import Context +from .context import Context if TYPE_CHECKING: from .. import dispatcher from bclib.utility import DictEx diff --git a/bclib/context/request_context.py b/bclib/context/request_context.py index d84e0c5..e4fd3a4 100644 --- a/bclib/context/request_context.py +++ b/bclib/context/request_context.py @@ -3,10 +3,10 @@ from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes from bclib.exception import ShortCircuitErr -from bclib.context.context import Context +from .context import Context if TYPE_CHECKING: - from .. import dispatcher + from bclib import dispatcher class RequestContext(Context): diff --git a/bclib/context/web_context.py b/bclib/context/web_context.py index 1793f78..6e2d24b 100644 --- a/bclib/context/web_context.py +++ b/bclib/context/web_context.py @@ -3,15 +3,15 @@ from typing import Any, TYPE_CHECKING, Coroutine, Iterator, Optional, Union from aiohttp.web_response import ContentCoding -from ..context.request_context import RequestContext +from .request_context import RequestContext if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher import dispatcher + from bclib.listener import WebMessage class WebContext(RequestContext): - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher',message_object: 'WebMessage') -> None: super().__init__(cms_object, dispatcher) self.process_async = True self.__message = message_object diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index 484625a..b586999 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -1,5 +1,6 @@ import asyncio +from context.context_factory import ContextFactory from dependency_injector import containers from bclib.logger import ILogger @@ -11,8 +12,8 @@ class DevServerDispatcher(RoutingDispatcher): - def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container=container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger = logger, loop=loop) + def __init__(self,container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container=container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger = logger, loop=loop) self.__listener = HttpListener( Endpoint(self.options.server), self._on_message_receive_async, diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index 579adb6..a623f97 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -12,13 +12,14 @@ from bclib.cache import CacheManager from bclib.listener import RabbitBusListener from bclib.predicate import Predicate -from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext, NamedPipeContext from bclib.db_manager import DbManager from bclib.utility import DictEx from bclib.exception import HandlerNotFoundErr from .callback_info import CallbackInfo +from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext + class Dispatcher(ABC): """Base class for dispatching request""" @@ -29,11 +30,7 @@ def __init__(self,container:'containers.Container', options: 'DictEx',cache_man self.event_loop: 'asyncio.AbstractEventLoop' = loop self.cache_manager: 'CacheManager' = cache_manager self.db_manager = db_manager - self.__logger: ILogger = logger - self.log_error: bool = self.options.log_error if self.options.has( - "log_error") else False - self.log_request: bool = self.options.log_request if self.options.has( - "log_request") else True + self.logger: ILogger = logger self.__rabbit_dispatcher: 'list[RabbitBusListener]' = list() if "router" in self.options and "rabbit" in self.options.router: for setting in self.options.router.rabbit: @@ -46,12 +43,12 @@ def socket_action(self, * predicates: (Predicate)): def _decorator(socket_action_handler: 'Callable[[SocketContext],bool]'): @wraps(socket_action_handler) - async def non_async_wrapper(context: SocketContext): + async def non_async_wrapper(context: 'SocketContext'): await self.event_loop.run_in_executor(None, socket_action_handler, context) return True @wraps(socket_action_handler) - async def async_wrapper(context: SocketContext): + async def async_wrapper(context: 'SocketContext'): await socket_action_handler(context) return True @@ -69,12 +66,12 @@ def restful_action(self, * predicates: (Predicate)): def _decorator(restful_action_handler: 'Callable[[RESTfulContext], dict]'): @wraps(restful_action_handler) - async def non_async_wrapper(context: RESTfulContext): + async def non_async_wrapper(context: 'RESTfulContext'): action_result = await self.event_loop.run_in_executor(None, restful_action_handler, context) return None if action_result is None else context.generate_response(action_result) @wraps(restful_action_handler) - async def async_wrapper(context: RESTfulContext): + async def async_wrapper(context: 'RESTfulContext'): action_result = await restful_action_handler(context) return None if action_result is None else context.generate_response(action_result) @@ -92,12 +89,12 @@ def web_action(self, * predicates: (Predicate)): def _decorator(web_action_handler: 'Callable[[WebContext], str]'): @wraps(web_action_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): action_result = await self.event_loop.run_in_executor(None, web_action_handler, context) return None if action_result is None else context.generate_response(action_result) @wraps(web_action_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): action_result = await web_action_handler(context) return None if action_result is None else context.generate_response(action_result) @@ -114,7 +111,7 @@ def client_source_action(self, *predicates: (Predicate)): def _decorator(client_source_action_handler: 'Callable[[ClientSourceContext], Any]'): @wraps(client_source_action_handler) - async def non_async_wrapper(context: ClientSourceContext): + async def non_async_wrapper(context: 'ClientSourceContext'): data = await self.event_loop.run_in_executor(None, client_source_action_handler, context) result_set = list() if data is not None: @@ -144,7 +141,7 @@ async def non_async_wrapper(context: ClientSourceContext): return None @wraps(client_source_action_handler) - async def async_wrapper(context: ClientSourceContext): + async def async_wrapper(context: 'ClientSourceContext'): data = await client_source_action_handler(context) result_set = list() if data is not None: @@ -188,11 +185,11 @@ def client_source_member_action(self, *predicates: (Predicate)): def _decorator(client_source_member_handler: 'Callable[[ClientSourceMemberContext], Any]'): @wraps(client_source_member_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): return await self.event_loop.run_in_executor(None, client_source_member_handler, context) @wraps(client_source_member_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): return await client_source_member_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -208,7 +205,7 @@ def server_source_action(self, *predicates: (Predicate)): def _decorator(server_source_action_handler: 'Callable[[ServerSourceContext], Any]'): @wraps(server_source_action_handler) - async def non_async_wrapper(context: ServerSourceContext): + async def non_async_wrapper(context: 'ServerSourceContext'): data = await self.event_loop.run_in_executor(None, server_source_action_handler, context) result_set = list() if data is not None: @@ -238,7 +235,7 @@ async def non_async_wrapper(context: ServerSourceContext): return None @wraps(server_source_action_handler) - async def async_wrapper(context: ServerSourceContext): + async def async_wrapper(context: 'ServerSourceContext'): data = await server_source_action_handler(context) result_set = list() if data is not None: @@ -282,11 +279,11 @@ def server_source_member_action(self, *predicates: (Predicate)): def _decorator(server_source_member_action_handler: 'Callable[[ServerSourceMemberContext], Any]'): @wraps(server_source_member_action_handler) - async def non_async_wrapper(context: WebContext): + async def non_async_wrapper(context: 'WebContext'): return await self.event_loop.run_in_executor(None, server_source_member_action_handler, context) @wraps(server_source_member_action_handler) - async def async_wrapper(context: WebContext): + async def async_wrapper(context: 'WebContext'): return await server_source_member_action_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -303,11 +300,11 @@ def rabbit_action(self, * predicates: (Predicate)): def _decorator(rabbit_action_handler: 'Callable[[RabbitContext], bool]'): @wraps(rabbit_action_handler) - async def non_async_wrapper(context: RabbitContext): + async def non_async_wrapper(context: 'RabbitContext'): return await self.event_loop.run_in_executor(None, rabbit_action_handler, context) @wraps(rabbit_action_handler) - async def async_wrapper(context: RabbitContext): + async def async_wrapper(context: 'RabbitContext'): return await rabbit_action_handler(context) wrapper = async_wrapper if inspect.iscoroutinefunction( @@ -319,28 +316,6 @@ async def async_wrapper(context: RabbitContext): return rabbit_action_handler return _decorator - def named_pipe_action(self, * predicates: (Predicate)): - """Decorator for determine named pipe message request action""" - - def _decorator(named_pipe_action_handler: 'Callable[[RabbitContext], bool]'): - - @wraps(named_pipe_action_handler) - async def non_async_wrapper(context: NamedPipeContext): - return await self.event_loop.run_in_executor(None, named_pipe_action_handler, context) - - @wraps(named_pipe_action_handler) - async def async_wrapper(context: NamedPipeContext): - return await named_pipe_action_handler(context) - - wrapper = async_wrapper if inspect.iscoroutinefunction( - named_pipe_action_handler) else non_async_wrapper - - self._get_context_lookup(NamedPipeContext.__name__)\ - .append(CallbackInfo([*predicates], wrapper)) - - return named_pipe_action_handler - return _decorator - def _get_context_lookup(self, key: str) -> 'list[CallbackInfo]': """Get key related action list object""" @@ -352,7 +327,7 @@ def _get_context_lookup(self, key: str) -> 'list[CallbackInfo]': self.__look_up[key] = ret_val return ret_val - async def dispatch_async(self, context: Context) -> Any: + async def dispatch_async(self, context: 'Context') -> Any: """Dispatch context and get result from related action method""" result: Any = None @@ -365,12 +340,10 @@ async def dispatch_async(self, context: Context) -> Any: break else: ex = HandlerNotFoundErr(name) - if self.log_error: - print(str(ex)) + self.logger._log_error(ex) result = context.generate_error_response(ex) except Exception as ex: - if self.log_error: - traceback.print_exc() + self.logger.log_error(ex) result = context.generate_error_response(ex) return result @@ -405,7 +378,7 @@ def listening(self, with_block:bool = True): self.event_loop.close() def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: - return self.__logger.new_object_log(schema_name, routing_key, **kwargs) + return self.logger.new_object_log(schema_name, routing_key, **kwargs) async def log_async(self, log_object: LogObject = None, **kwargs): """log params""" @@ -414,7 +387,7 @@ async def log_async(self, log_object: LogObject = None, **kwargs): raise Exception("'schema_name' not set for apply logging!") schema_name = kwargs.pop("schema_name") log_object = self.new_object_log(schema_name, **kwargs) - await self.__logger.log_async(log_object) + await self.logger.log_async(log_object) def log_in_background(self, log_object: LogObject = None, **kwargs) -> Coroutine: """log params in background precess""" diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index 8d2fb55..5f87d2a 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -1,5 +1,6 @@ import asyncio +from context.context_factory import ContextFactory from dependency_injector import containers from bclib.cache import CacheManager from bclib.db_manager import DbManager @@ -11,8 +12,8 @@ class EndpointDispatcher(RoutingDispatcher): - def __init__(self, container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) + def __init__(self, container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container= container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) self.__endpoint = Endpoint(self.options.endpoint) def initialize_task(self): diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index 4f45f04..1cc7de1 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -1,8 +1,7 @@ import asyncio import inspect import json -import re -from struct import error + from typing import Callable, Any, Coroutine, Optional from dependency_injector import containers @@ -10,7 +9,7 @@ from bclib.db_manager import DbManager from bclib.cache import CacheManager from bclib.utility import DictEx -from bclib.context import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext, NamedPipeContext +from bclib.context.context_factory import ContextFactory from bclib.listener import Message, MessageType, HttpBaseDataType, ReceiveMessage from .dispatcher_helper import DispatcherHelper from .dispatcher import Dispatcher @@ -18,69 +17,15 @@ class RoutingDispatcher(Dispatcher, DispatcherHelper): - def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + def __init__(self,container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) - self.__default_router = self.options.defaultRouter\ - if 'defaultRouter' in self.options and isinstance(self.options.defaultRouter, str)\ - else None - self.name = self.options["name"] if self.options.has("name") else None - self.__log_name = f"{self.name}: " if self.name else '' - if self.options.has('router'): - router = self.options.router - if isinstance(router, str): - self.__context_type_detector: 'Callable[[str],str]' = lambda _: router - elif isinstance(router, DictEx): - self.init_router_lookup() - else: - raise error( - "Invalid value for 'router' property in host options! Use string or dict object only.") - elif self.__default_router: - self.__context_type_detector: 'Callable[[str],str]' = lambda _: self.__default_router - else: - raise error( - "Invalid routing config! Please at least set one of 'router' or 'defaultRouter' property in host options.") - - def init_router_lookup(self): - """create router lookup dictionary""" - - route_dict = dict() - for key, values in self.options.router.items(): - if key != 'rabbit'.strip(): - if '*' in values: - route_dict['*'] = key - break - else: - for value in values: - if len(value.strip()) != 0 and value not in route_dict: - route_dict[value] = key - if len(route_dict) == 1 and '*' in route_dict and self.__default_router is None: - router = route_dict['*'] - self.__context_type_detector: 'Callable[[str],str]' = lambda _: router - else: - self.__context_type_lookup = route_dict.items() - self.__context_type_detector = self.__context_type_detect_from_lookup - - def __context_type_detect_from_lookup(self, url: str) -> str: - """Detect context type from url about lookup""" - - context_type: str = None - if url: - try: - for pattern, lookup_context_type in self.__context_type_lookup: - if pattern == "*" or re.search(pattern, url): - context_type = lookup_context_type - break - except TypeError: - pass - except error as ex: - print("Error in detect context from routing options!", ex) - return context_type if context_type else self.__default_router + self._context_factory = context_factory async def _on_message_receive_async(self, message: Message) -> Message: """Process received message""" try: - context = self.__context_factory(message) + context = self._context_factory.create_context(message,self) response = await self.dispatch_async(context) ret_val: Message = None if context.is_adhoc: @@ -93,60 +38,6 @@ async def _on_message_receive_async(self, message: Message) -> Message: print(f"Error in process received message {ex}") raise ex - def __context_factory(self, message: Message) -> Context: - """Create context from message object""" - - ret_val: RequestContext = None - context_type = None - cms_object: Optional[dict] = None - url: Optional[str] = None - request_id: Optional[str] = None - method: Optional[str] = None - message_json: Optional[dict] = None - if message.buffer is not None: - message_json = json.loads(message.buffer) - cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None - if cms_object: - if 'request' in cms_object: - req = cms_object["request"] - else: - raise KeyError("request key not found in cms object") - if 'full-url' in req: - url = req["full-url"] - else: - raise KeyError("full-url key not found in request") - request_id = req['request-id'] if 'request-id' in req else 'none' - method = req['methode'] if 'methode' in req else 'none' - if message.type == MessageType.AD_HOC: - if url or self.__default_router is None: - context_type = self.__context_type_detector(url) - else: - context_type = self.__default_router - else: - context_type = "socket" - if self.log_request: - print( - f"{self.__log_name}({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") - - if context_type == "client_source": - ret_val = ClientSourceContext(cms_object, self,message) - elif context_type == "restful": - ret_val = RESTfulContext(cms_object, self,message) - elif context_type == "server_source": - ret_val = ServerSourceContext(message_json, self) - elif context_type == "web": - ret_val = WebContext(cms_object, self,message) - elif context_type == "socket": - ret_val = SocketContext(cms_object, self, message, message_json) - elif context_type == "named_pipe": - ret_val = NamedPipeContext(message_json, message.buffer.decode("utf-8"), self) - elif context_type is None: - raise NameError(f"No context found for '{url}'") - else: - raise NameError( - f"Configured context type '{context_type}' not found for '{url}'") - return ret_val - def run_in_background(self, callback: 'Callable|Coroutine', *args: Any) -> asyncio.Future: """helper for run function in background thread""" diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index 84befcc..dd0d205 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -1,4 +1,5 @@ import asyncio +from context.context_factory import ContextFactory from dependency_injector import containers from bclib.cache import CacheManager @@ -10,8 +11,8 @@ class SocketDispatcher(RoutingDispatcher): - def __init__(self, container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) + def __init__(self, container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): + super().__init__(container= container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) self.__lock = asyncio.Lock() self.__listener = SocketListener( Endpoint(self.options.receiver), diff --git a/bclib/edge.py b/bclib/edge.py index 053df95..3584f5d 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -4,7 +4,7 @@ from dependency_injector import providers from bclib.db_manager import * from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher -from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, NamedPipeContext +from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate diff --git a/bclib/edge_container.py b/bclib/edge_container.py index 0560785..3724eac 100644 --- a/bclib/edge_container.py +++ b/bclib/edge_container.py @@ -7,6 +7,7 @@ from bclib.cache import CacheFactory from bclib.utility import DictEx from bclib.dispatcher import SocketDispatcher, DevServerDispatcher, EndpointDispatcher +from bclib.context.context_factory import ContextFactory def get_mode(options:'DictEx'): if options.has("server"): @@ -38,9 +39,10 @@ class EdgeContainer(containers.DeclarativeContainer): app_cache_manager = providers.Singleton(CacheFactory.create,app_cache_options) app_db_manager = providers.Singleton(DbManager,app_options, app_event_loop) app_logger= providers.Singleton(LoggerFactory.create,app_options) - app_server_dispatcher = providers.Singleton(DevServerDispatcher,app_container, app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) - app_endpoint_dispatcher = providers.Singleton(EndpointDispatcher,app_container,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) - app_socket_dispatcher = providers.Singleton(SocketDispatcher,app_container,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_context_factory = providers.Singleton(ContextFactory,app_options,app_logger) + app_server_dispatcher = providers.Singleton(DevServerDispatcher,app_container,app_context_factory, app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_endpoint_dispatcher = providers.Singleton(EndpointDispatcher,app_container,app_context_factory,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_socket_dispatcher = providers.Singleton(SocketDispatcher,app_container,app_context_factory,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) dispatcher = providers.Selector( app_mode, diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 1cd2581..4ef7f23 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -11,10 +11,9 @@ from bclib.listener.message_type import MessageType from ..endpoint import Endpoint -from ..http_listener.http_base_data_name import HttpBaseDataName -from ..http_listener.http_base_data_type import HttpBaseDataType +from .http_base_data_name import HttpBaseDataName +from .http_base_data_type import HttpBaseDataType from bclib.utility import DictEx, ResponseTypes -from ..message import Message from ..web_message import WebMessage import pathlib diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index a3d8a61..2d01e09 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -1,11 +1,11 @@ from struct import error from bclib.context import RabbitContext from typing import TYPE_CHECKING -from bclib.listener.rabbit_listener import RabbitListener +from .rabbit_listener import RabbitListener from bclib.utility import DictEx if TYPE_CHECKING: - from .. import dispatcher + from bclib import dispatcher class RabbitBusListener(RabbitListener): diff --git a/bclib/logger/exchange_rabbit_schema_base_logger.py b/bclib/logger/exchange_rabbit_schema_base_logger.py index 963eb68..f08ae0e 100644 --- a/bclib/logger/exchange_rabbit_schema_base_logger.py +++ b/bclib/logger/exchange_rabbit_schema_base_logger.py @@ -28,5 +28,5 @@ def send_to_rabbit(options): channel.basic_publish( exchange='', routing_key=queue, body=json.dumps(schema, ensure_ascii=False)) loop = asyncio.get_running_loop() - future = loop.run_in_executor(None, send_to_rabbit, self.options) + future = loop.run_in_executor(None, send_to_rabbit, self.options.logger) await future diff --git a/bclib/logger/ilogger.py b/bclib/logger/ilogger.py index d6813a2..13764b7 100644 --- a/bclib/logger/ilogger.py +++ b/bclib/logger/ilogger.py @@ -1,10 +1,24 @@ from abc import ABC, abstractmethod -from .log_object import LogObject + +import traceback from typing import Optional +from bclib.utility import DictEx +from .log_object import LogObject class ILogger(ABC): """Base class for logger""" + def __init__(self,options:'DictEx'): + self.options = options + self.name = options["name"] if options.has("name") else None + self.__log_name = f"{self.name}: " if self.name else '' + self._log_error: bool = self.options.log_error if self.options.has( + "log_error") else False + self.__log_request: bool = self.options.log_request if self.options.has( + "log_request") else True + print(f'{self.__class__.__name__} start logging') + + @abstractmethod async def log_async(self, log_object: LogObject): """log data async""" @@ -12,3 +26,12 @@ async def log_async(self, log_object: LogObject): def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: """New object log""" return LogObject(schema_name, routing_key, **kwargs) + + def log_request(self, message:'str'): + if(self.__log_request): + print(self.__log_name, message) + + def log_error(self,error:'Exception'): + print(self.__log_name,str(error)) + traceback.print_exc() + diff --git a/bclib/logger/logger_factory.py b/bclib/logger/logger_factory.py index 97e7034..41a6abd 100644 --- a/bclib/logger/logger_factory.py +++ b/bclib/logger/logger_factory.py @@ -1,17 +1,18 @@ +from typing import Optional from bclib.utility import DictEx -from ..logger.rabbit_schema_base_logger import RabbitSchemaBaseLogger -from ..logger.restful_schema_base_logger import RESTfulSchemaBaseLogger -from ..logger.no_logger import NoLogger -from ..logger.ilogger import ILogger +from .rabbit_schema_base_logger import RabbitSchemaBaseLogger +from .restful_schema_base_logger import RESTfulSchemaBaseLogger +from .no_logger import NoLogger +from .ilogger import ILogger class LoggerFactory: @staticmethod def create(options: DictEx) -> ILogger: - logger: ILogger = None + logger: Optional[ILogger] = None if not options.has("logger"): - logger = NoLogger() + logger = NoLogger(options) else: logger_option: DictEx = options.logger if not logger_option.has('type'): @@ -25,5 +26,4 @@ def create(options: DictEx) -> ILogger: else: raise Exception( f"Type '{logger_type}' not support for logger") - print(f'{logger.__class__.__name__} start logging') return logger diff --git a/bclib/logger/no_logger.py b/bclib/logger/no_logger.py index b5474dc..686ae20 100644 --- a/bclib/logger/no_logger.py +++ b/bclib/logger/no_logger.py @@ -1,9 +1,13 @@ -from bclib.logger.log_object import LogObject -from ..logger.ilogger import ILogger +from bclib.utility import DictEx +from .log_object import LogObject +from .ilogger import ILogger class NoLogger(ILogger): """class for no logging""" + def __init__(self,options:'DictEx'): + super().__init__(options) + async def log_async(self, log_object: LogObject): """log data async""" \ No newline at end of file diff --git a/bclib/logger/rabbit_schema_base_logger.py b/bclib/logger/rabbit_schema_base_logger.py index 982332b..02ff568 100644 --- a/bclib/logger/rabbit_schema_base_logger.py +++ b/bclib/logger/rabbit_schema_base_logger.py @@ -6,11 +6,11 @@ class RabbitSchemaBaseLogger(SchemaBaseLogger): - def __init__(self, options: DictEx) -> None: + def __init__(self, options: 'DictEx') -> None: super().__init__(options) - if "connection" not in options: + if "connection" not in options.logger: raise Exception("connection not set in logger option.") - self.__connection_options = options.connection + self.__connection_options = options.logger.connection if "url" not in self.__connection_options: raise Exception("url not set in connection option.") if "queue" in self.__connection_options and "exchange" in self.__connection_options: diff --git a/bclib/logger/restful_schema_base_logger.py b/bclib/logger/restful_schema_base_logger.py index b6b6f9d..554f243 100644 --- a/bclib/logger/restful_schema_base_logger.py +++ b/bclib/logger/restful_schema_base_logger.py @@ -3,12 +3,12 @@ class RESTfulSchemaBaseLogger(SchemaBaseLogger): - def __init__(self, options: DictEx) -> None: + def __init__(self, options: 'DictEx') -> None: super().__init__(options) - if options.has("url"): - self.__post_url = options.url - elif options.has("post_url"): - self.__post_url = options.post_url + if options.logger.has("url"): + self.__post_url = options.logger.url + elif options.logger.has("post_url"): + self.__post_url = options.logger.post_url else: raise Exception( "url part of schema logger not set. set 'url' or 'post_url'") diff --git a/bclib/logger/schema_base_logger.py b/bclib/logger/schema_base_logger.py index cc2f5ba..d4e2707 100644 --- a/bclib/logger/schema_base_logger.py +++ b/bclib/logger/schema_base_logger.py @@ -9,13 +9,12 @@ class SchemaBaseLogger(ILogger): - def __init__(self, options: DictEx) -> None: - super().__init__() - self.options = options - if options.has("url"): - self.__get_url = options.url - elif options.has("get_url"): - self.__get_url = options.get_url + def __init__(self, options: 'DictEx') -> None: + super().__init__(options) + if options.logger.has("url"): + self.__get_url = options.logger.url + elif options.logger.has("get_url"): + self.__get_url = options.logger.get_url else: raise Exception( "url part of schema logger not set. set 'url' or 'get_url'") diff --git a/test/dev_server/simple.py b/test/dev_server/simple.py index a1b0ad1..bec1ad8 100644 --- a/test/dev_server/simple.py +++ b/test/dev_server/simple.py @@ -19,7 +19,7 @@ async def manage_resource2_async(): @containers.copy(EdgeContainer) class AppContainer(EdgeContainer): app_resource1 = providers.Resource(manage_resource1) - app_resource2 = providers.Resource(manage_resource2_async) + #app_resource2 = providers.Resource(manage_resource2_async) object_provider = providers.Callable(lambda: random.randint(100,999)) object_val = providers.Callable(lambda: random.randint(100,999)) object_val2 = providers.Callable(lambda: random.randint(1000,9999)) @@ -44,7 +44,7 @@ def process_web_action(context: edge.WebContext): @app.web_action() @inject -def process_default_web_action(context: edge.WebContext,val=Provide["object_provider"],container:AppContainer =Provide["edge_container"]): +def process_default_web_action(context: edge.WebContext,val=Provide["object_provider"],container:AppContainer =Provide["app_container"]): return "result from process_default_web_action " + str(val or "?") +" "+ str(context.dispatcher.container.object_val()) + " "+ str(container.object_val2()) From 45ba70086aa65be46d2750bad659ebdc949581ba Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Wed, 4 Dec 2024 21:46:22 +0330 Subject: [PATCH 05/15] update message class and related subclasses --- bclib/context/context_factory.py | 106 +++++++--- bclib/context/rabbit_context.py | 4 +- bclib/context/socket_context.py | 20 +- bclib/context/web_context.py | 1 - bclib/dispatcher/dispatcher.py | 2 +- bclib/dispatcher/endpoint_dispatcher.py | 11 +- bclib/dispatcher/routing_dispatcher.py | 22 +-- bclib/edge.py | 75 +++---- bclib/listener/__init__.py | 1 + bclib/listener/http_listener/http_listener.py | 178 +---------------- bclib/listener/message.py | 29 ++- bclib/listener/rabbit_bus_listener.py | 6 +- bclib/listener/rabbit_listener.py | 2 +- bclib/listener/receive_message.py | 59 +++++- bclib/listener/socket_listener.py | 16 +- bclib/listener/web_message.py | 185 +++++++++++++++++- bclib/logger/ilogger.py | 4 +- test/dev_server/simple.py | 36 ++-- test/web/simple.py | 6 +- test/web_socket/list-data.py | 6 +- 20 files changed, 454 insertions(+), 315 deletions(-) diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py index 2004dc2..b00cd1a 100644 --- a/bclib/context/context_factory.py +++ b/bclib/context/context_factory.py @@ -1,7 +1,7 @@ import json import re from struct import error -from typing import Optional,TYPE_CHECKING +from typing import Optional, TYPE_CHECKING if TYPE_CHECKING: @@ -11,7 +11,6 @@ from . import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext - # from .client_source_context import ClientSourceContext # from .restful_context import RESTfulContext # from .web_context import WebContext @@ -23,8 +22,9 @@ from bclib.listener.message import Message, MessageType from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + class ContextFactory: - def __init__(self,options:'DictEx',logger:'ILogger'): + def __init__(self, options: 'DictEx', logger: 'ILogger'): self.logger = logger self.__default_router = options.defaultRouter\ if 'defaultRouter' in options and isinstance(options.defaultRouter, str)\ @@ -44,7 +44,7 @@ def __init__(self,options:'DictEx',logger:'ILogger'): raise error( "Invalid routing config! Please at least set one of 'router' or 'defaultRouter' property in host options.") - def __init_router_lookup(self,router:'DictEx'): + def __init_router_lookup(self, router: 'DictEx'): """create router lookup dictionary""" route_dict = dict() @@ -81,32 +81,30 @@ def __context_type_detect_from_lookup(self, url: str) -> str: print("Error in detect context from routing options!", ex) return context_type if context_type else self.__default_router - - def create_context(self, message: 'Message', dispatcher:'IDispatcher') -> Context: + async def create_context_async(self, message: 'Message', dispatcher: 'IDispatcher') -> Context: """Create context from message object""" - print('qam',message) + message_json: dict = await message.get_json_async() + ret_val: RequestContext = None context_type = None cms_object: Optional[dict] = None url: Optional[str] = None request_id: Optional[str] = None method: Optional[str] = None - message_json: Optional[dict] = None - if message.buffer is not None: - message_json = json.loads(message.buffer) - cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None - if cms_object: - if 'request' in cms_object: - req = cms_object["request"] - else: - raise KeyError("request key not found in cms object") - if 'full-url' in req: - url = req["full-url"] - else: - raise KeyError("full-url key not found in request") - request_id = req['request-id'] if 'request-id' in req else 'none' - method = req['methode'] if 'methode' in req else 'none' + + cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None + if cms_object: + if 'request' in cms_object: + req = cms_object["request"] + else: + raise KeyError("request key not found in cms object") + if 'full-url' in req: + url = req["full-url"] + else: + raise KeyError("full-url key not found in request") + request_id = req['request-id'] if 'request-id' in req else 'none' + method = req['methode'] if 'methode' in req else 'none' if message.type == MessageType.AD_HOC: if url or self.__default_router is None: context_type = self.__context_type_detector(url) @@ -115,21 +113,75 @@ def create_context(self, message: 'Message', dispatcher:'IDispatcher') -> Contex else: context_type = "socket" self.logger.log_request( - f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") + f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") if context_type == "client_source": - ret_val = ClientSourceContext(cms_object, dispatcher,message) + ret_val = ClientSourceContext(cms_object, dispatcher, message) elif context_type == "restful": - ret_val = RESTfulContext(cms_object, dispatcher,message) + ret_val = RESTfulContext(cms_object, dispatcher, message) elif context_type == "server_source": ret_val = ServerSourceContext(message_json, dispatcher) elif context_type == "web": - ret_val = WebContext(cms_object, dispatcher,message) + ret_val = WebContext(cms_object, dispatcher, message) elif context_type == "socket": - ret_val = SocketContext(cms_object, dispatcher, message, message_json) + ret_val = SocketContext( + cms_object, dispatcher, message, message_json) elif context_type is None: raise NameError(f"No context found for '{url}'") else: raise NameError( f"Configured context type '{context_type}' not found for '{url}'") return ret_val + + # async def create_context_async(self, message: 'Message', dispatcher:'IDispatcher') -> Context: + # """Create context from message object""" + + # print('qam',json.dumps( message)) + # ret_val: RequestContext = None + # context_type = None + # cms_object: Optional[dict] = None + # url: Optional[str] = None + # request_id: Optional[str] = None + # method: Optional[str] = None + # message_json: Optional[dict] = None + # if True: #message.buffer is not None: + # #message_json = json.loads(message.buffer) + # message_json = await message.get_json_async() + # cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None + # if cms_object: + # if 'request' in cms_object: + # req = cms_object["request"] + # else: + # raise KeyError("request key not found in cms object") + # if 'full-url' in req: + # url = req["full-url"] + # else: + # raise KeyError("full-url key not found in request") + # request_id = req['request-id'] if 'request-id' in req else 'none' + # method = req['methode'] if 'methode' in req else 'none' + # if message.type == MessageType.AD_HOC: + # if url or self.__default_router is None: + # context_type = self.__context_type_detector(url) + # else: + # context_type = self.__default_router + # else: + # context_type = "socket" + # self.logger.log_request( + # f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") + + # if context_type == "client_source": + # ret_val = ClientSourceContext(cms_object, dispatcher,message) + # elif context_type == "restful": + # ret_val = RESTfulContext(cms_object, dispatcher,message) + # elif context_type == "server_source": + # ret_val = ServerSourceContext(message_json, dispatcher) + # elif context_type == "web": + # ret_val = WebContext(cms_object, dispatcher,message) + # elif context_type == "socket": + # ret_val = SocketContext(cms_object, dispatcher, message, message_json) + # elif context_type is None: + # raise NameError(f"No context found for '{url}'") + # else: + # raise NameError( + # f"Configured context type '{context_type}' not found for '{url}'") + # return ret_val diff --git a/bclib/context/rabbit_context.py b/bclib/context/rabbit_context.py index 39edb97..411f13d 100644 --- a/bclib/context/rabbit_context.py +++ b/bclib/context/rabbit_context.py @@ -2,14 +2,14 @@ from typing import Any, TYPE_CHECKING from .context import Context if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher import IDispatcher from bclib.utility import DictEx class RabbitContext(Context): """Context for rabbit-mq request""" - def __init__(self, rabbit_message: DictEx, dispatcher: 'dispatcher.IDispatcher'): + def __init__(self, rabbit_message: 'DictEx', dispatcher: 'IDispatcher'): super().__init__(dispatcher) self.__rabbit_message: DictEx = rabbit_message self.__message = DictEx(json.loads( diff --git a/bclib/context/socket_context.py b/bclib/context/socket_context.py index 024e3f6..5f0ba72 100644 --- a/bclib/context/socket_context.py +++ b/bclib/context/socket_context.py @@ -1,6 +1,8 @@ import json from typing import TYPE_CHECKING -from .context import Context + +from bclib.listener.message_type import MessageType +from bclib.context import Context from bclib.listener.receive_message import ReceiveMessage from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, HttpStatusCodes, ResponseTypes @@ -12,7 +14,7 @@ class SocketContext(Context): """Base class for dispatching web socket base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher', message_object: 'listener.ReceiveMessage', body: dict) -> None: + def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher', message_object: 'listener.SocketMessage', body: dict) -> None: super().__init__(dispatcher) self.cms = DictEx(cms_object) if cms_object else None self.url: str = self.cms.request.url if cms_object else None @@ -38,13 +40,15 @@ async def send_content_async(self, headers: 'dict' = None) -> None: cms = Context._generate_response_cms( content, response_type, status_code, mime, template, headers) - message = ReceiveMessage.create_from_object( - self.message.session_id, cms) - await message.write_to_stream_async(self.message.writer) + await self.message.write_result_async(cms,MessageType.MESSAGE) + # message = ReceiveMessage.create_from_object( + # self.message.session_id, cms) + # await message.write_to_stream_async(self.message.writer) - async def read_message_async(self) -> 'listener.ReceiveMessage': + async def read_message_async(self) -> 'listener.SocketMessage': return await self.message.read_next_message_async() async def send_close_async(self) -> None: - message = ReceiveMessage.create_disconnect(self.message.session_id) - await message.write_to_stream_async(self.message.writer) + #message = ReceiveMessage.create_disconnect(self.message.session_id) + #await message.write_to_stream_async(self.message.writer) + await self.message.write_result_async(None,MessageType.DISCONNECT) diff --git a/bclib/context/web_context.py b/bclib/context/web_context.py index 6e2d24b..f4c3c61 100644 --- a/bclib/context/web_context.py +++ b/bclib/context/web_context.py @@ -6,7 +6,6 @@ from .request_context import RequestContext if TYPE_CHECKING: - from bclib.dispatcher import dispatcher from bclib.listener import WebMessage diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index a623f97..29f09e2 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -340,7 +340,7 @@ async def dispatch_async(self, context: 'Context') -> Any: break else: ex = HandlerNotFoundErr(name) - self.logger._log_error(ex) + self.logger.log_error(ex) result = context.generate_error_response(ex) except Exception as ex: self.logger.log_error(ex) diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index 5f87d2a..2f4a5fa 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -2,6 +2,7 @@ from context.context_factory import ContextFactory from dependency_injector import containers +from listener.receive_message import SocketMessage from bclib.cache import CacheManager from bclib.db_manager import DbManager from bclib.logger import ILogger @@ -12,8 +13,9 @@ class EndpointDispatcher(RoutingDispatcher): - def __init__(self, container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container= container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__endpoint = Endpoint(self.options.endpoint) def initialize_task(self): @@ -21,9 +23,8 @@ def initialize_task(self): async def on_connection_open(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): try: - msg = await ReceiveMessage.read_from_stream_async(reader, writer) - result = await self._on_message_receive_async(msg) - await result.write_to_stream_async(writer) + msg = SocketMessage(reader, writer) + await self._on_message_receive_async(message=msg) except: pass try: diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index 1cc7de1..1296915 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -12,28 +12,24 @@ from bclib.context.context_factory import ContextFactory from bclib.listener import Message, MessageType, HttpBaseDataType, ReceiveMessage from .dispatcher_helper import DispatcherHelper -from .dispatcher import Dispatcher +from .dispatcher import Dispatcher class RoutingDispatcher(Dispatcher, DispatcherHelper): - def __init__(self,container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container= container, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self._context_factory = context_factory - async def _on_message_receive_async(self, message: Message) -> Message: + async def _on_message_receive_async(self, message: Message): """Process received message""" try: - context = self._context_factory.create_context(message,self) + await message.fill_async() + context = await self._context_factory.create_context_async(message, self) response = await self.dispatch_async(context) - ret_val: Message = None - if context.is_adhoc: - ret_val = message.create_response_message( - message.session_id, - json.dumps(response, ensure_ascii=False).encode("utf-8") - ) - return ret_val + await message.set_result_async(response) except Exception as ex: print(f"Error in process received message {ex}") raise ex @@ -51,7 +47,7 @@ async def send_message_async(self, message: MessageType) -> bool: raise NotImplementedError( "Send ad-hoc message not support in this type of dispatcher") - def cache(self, life_time:"int"=0, key:"str"=None): + def cache(self, life_time: "int" = 0, key: "str" = None): """Cache result of function for seconds of time or until signal by key for clear""" return self.cache_manager.cache_decorator(key, life_time) diff --git a/bclib/edge.py b/bclib/edge.py index 3584f5d..82853d3 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -40,39 +40,47 @@ def from_list(hosts: 'dict[str,list[str]]'): loop.run_until_complete(asyncio.gather(*tasks)) -def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: - """Create related RoutingDispatcher obj from config object""" - - import sys - import getopt - multi: bool = False - argument_list = sys.argv[1:] - # Options - short_options = "mn:" - # Long options - long_options = ["Name =", "Multi"] - try: - arguments, _ = getopt.gnu_getopt( - argument_list, short_options, long_options) - for current_argument, current_value in arguments: - if current_argument in ("-n", "--Name"): - options["name"] = current_value.strip() - elif current_argument in ("-m", "--Multi"): - multi = True - except getopt.error as err: - print(str(err)) +# def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: +# """Create related RoutingDispatcher obj from config object""" + +# import sys +# import getopt + +# multi: bool = False +# argument_list = sys.argv[1:] +# # Options +# short_options = "mn:" +# # Long options +# long_options = ["Name =", "Multi"] +# try: +# arguments, _ = getopt.gnu_getopt( +# argument_list, short_options, long_options) +# for current_argument, current_value in arguments: +# if current_argument in ("-n", "--Name"): +# options["name"] = current_value.strip() +# elif current_argument in ("-m", "--Multi"): +# multi = True +# except getopt.error as err: +# print(str(err)) + +# if not multi: +# __print_splash(False) +# ret_val: RoutingDispatcher = None +# if "server" in options: +# ret_val = DevServerDispatcher(options=options,loop=loop) +# elif "endpoint" in options: +# ret_val = EndpointDispatcher(options=options,loop=loop) +# else: +# ret_val = SocketDispatcher(options=options,loop=loop) +# return ret_val - if not multi: - __print_splash(False) - ret_val: RoutingDispatcher = None - if "server" in options: - ret_val = DevServerDispatcher(options=options,loop=loop) - elif "endpoint" in options: - ret_val = EndpointDispatcher(options=options,loop=loop) - else: - ret_val = SocketDispatcher(options=options,loop=loop) - return ret_val +def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: + container=EdgeContainer() + container.app_config.from_dict(options) + if loop: + container.app_config.loop.from_value(loop) + return from_container(container) def __get_arg_parts(container:'EdgeContainer'): import sys @@ -96,10 +104,11 @@ def __get_arg_parts(container:'EdgeContainer'): except getopt.error as err: print(str(err)) -def create_server(container:'EdgeContainer') -> RoutingDispatcher: +def from_container(container:'EdgeContainer') -> RoutingDispatcher: """Create related RoutingDispatcher obj from config object""" - container.app_container.override(providers.Object(container)) + if type(container) is not EdgeContainer: + container.app_container.override(providers.Object(container)) __get_arg_parts(container) if not container.app_config.is_multi(): __print_splash(False) diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index ebf9e3f..8b21f36 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -8,3 +8,4 @@ from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.web_message import WebMessage +from bclib.listener.receive_message import SocketMessage diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 4ef7f23..816a655 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -1,15 +1,8 @@ import asyncio -import cgi -import io -import datetime import json -import uuid import ssl import base64 from typing import Callable, TYPE_CHECKING, Optional, Awaitable -from urllib.parse import unquote, parse_qs - -from bclib.listener.message_type import MessageType from ..endpoint import Endpoint from .http_base_data_name import HttpBaseDataName from .http_base_data_type import HttpBaseDataType @@ -56,55 +49,9 @@ async def __server_task(self, event_loop: asyncio.AbstractEventLoop): from aiohttp import web from multidict import MultiDict async def on_request_receive_async(request: 'web.Request') -> web.Response: - ret_val: web.Response = None - request_cms = await self.create_cms_async(request) - msg = WebMessage(request, str(uuid.uuid4()),MessageType.AD_HOC,json.dumps(request_cms, ensure_ascii=False).encode(encoding="utf-8")) - result = await self.on_message_receive_async(msg) - if result and result.Response is None: - cms: dict = json.loads(result.buffer.decode("utf-8")) - cms_cms = cms[HttpBaseDataType.CMS] - cms_cms_webserver = cms_cms[HttpBaseDataType.WEB_SERVER] - index = cms_cms_webserver[HttpBaseDataName.INDEX] - header_code: str = cms_cms_webserver[HttpBaseDataName.HEADER_CODE] - mime = cms_cms[HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] - headers = MultiDict() - if HttpBaseDataName.HTTP in cms_cms: - http: dict = cms_cms[HttpBaseDataName.HTTP] - for key, value in http.items(): - if isinstance(value, list): - for item in value: - headers.add(key, item) - else: - headers.add(key, value) - headers.add("Content-Type", mime) - if index == ResponseTypes.STATIC_FILE: - try: - path = pathlib.Path(cms_cms_webserver[HttpBaseDataName.FILE_PATH]) - path.stat() - ret_val = web.FileResponse( - path=path, - chunk_size=256*1024, - status=int(header_code.split(' ')[0]), - headers=headers - ) - except FileNotFoundError: - ret_val = web.Response( - status=404, - reason="File not found" - ) - else: - ret_val = web.Response( - status=int(header_code.split(' ')[0]), - headers=headers - ) - if HttpBaseDataName.CONTENT in cms_cms: - ret_val.text = cms_cms[HttpBaseDataName.CONTENT] - else: - raw_blob_content = cms_cms[HttpBaseDataName.BLOB_CONTENT] - ret_val.body = base64.b64decode(raw_blob_content.encode("utf-8")) - else: - ret_val = web.Response() if result.Response is None else result.Response - return ret_val + msg = WebMessage(request) + await self.on_message_receive_async(msg) + return msg.Response app = web.Application( logger=self.__logger, @@ -165,121 +112,4 @@ def convert_pfx_to_pem_file(pfxfile:str,password:str)->str: pem_file.write(private_key.private_bytes(encoding= Encoding.PEM,format=PrivateFormat.TraditionalOpenSSL,encryption_algorithm=NoEncryption())) return pem_file_path except Exception as ex: - raise Exception("Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) - - @staticmethod - async def create_cms_async(request: 'web.Request') -> dict: - cms_object = dict() - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.METHODE, request.method.lower()) - raw_url = unquote(request.path_qs)[1:] - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.RAW_URL, raw_url) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.URL, request.path[1:]) - HttpListener.__add_query_string(request.query, cms_object) - for key, value in request.headers.items(): - field_name = key.strip().lower() - if field_name == HttpBaseDataName.COOKIE: - HttpListener.__add_cookie(value, cms_object) - elif field_name == HttpBaseDataName.HOST: - HttpListener.__add_host(value, raw_url, cms_object) - else: - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - field_name, str(value).strip()) - HttpListener.__add_server_data(cms_object, request) - await HttpListener.__add_body_async(cms_object, request) - return {"cms": cms_object} - - @staticmethod - async def __add_body_async(cms_object: dict, request: 'web.Request'): - content_len_str = request.headers.get('Content-Length') - if content_len_str or request.can_read_body: - raw_body = await request.read() - body = raw_body.decode('utf-8') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.BODY, body) - content_type: str = request.headers.get( - "content-type") - if content_type and content_type.find("application/json") < 0: - if content_type.find("multipart/form-data") >= 0: - _, content_type_value_params = cgi.parse_header( - content_type) - content_type_value_params['boundary'] = bytes( - content_type_value_params['boundary'], "utf-8") - if content_len_str is not None: - content_type_value_params['CONTENT-LENGTH'] = int( - content_len_str) - with io.BytesIO(raw_body) as stream: - fields = cgi.parse_multipart( - stream, content_type_value_params) - for key, value in fields.items(): - HttpListener.__add_header(cms_object, - HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) - else: - for key, value in parse_qs(body).items(): - HttpListener.__add_header( - cms_object, HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) - - @staticmethod - def __add_server_data(cms_object: dict, request: 'web.Request'): - HttpListener._id += 1 - now = datetime.datetime.now() - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE, now.strftime("%d/%m/%Y")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.TIME, now.strftime("%H:%M")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE2, now.strftime("%Y%m%d")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.TIME2, now.strftime("%H%M%S")) - HttpListener.__add_header(cms_object, HttpBaseDataType.CMS, - HttpBaseDataName.DATE3, now.strftime("%Y.%m.%d")) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.REQUEST_ID, str(HttpListener._id)) - host_parts = request.host.split(':') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST_IP, host_parts[0]) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST_PORT, host_parts[1] if len(host_parts) > 1 else "80") # edit - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.CLIENT_IP, str(request.remote)) - - @staticmethod - def __add_query_string(query: dict, cms_object: dict) -> None: - for key, value in query.items(): - HttpListener.__add_header( - cms_object, HttpBaseDataType.QUERY, key, value) - - @staticmethod - def __add_cookie(raw_header_value: str, cms_object) -> None: - for item in raw_header_value.split(';'): - parts = item.split('=') - if len(parts) == 2: - HttpListener.__add_header(cms_object, HttpBaseDataName.COOKIE, - parts[0].strip(), parts[1].strip()) - - @staticmethod - def __add_host(row_host: str, row_url: str, cms_object) -> None: - host_parts = row_host.split(':') - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.HOST, host_parts[0]) - if len(host_parts) == 2: - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.PORT, host_parts[1]) - HttpListener.__add_header(cms_object, HttpBaseDataType.REQUEST, - HttpBaseDataName.FULL_URL, f"{row_host}/{row_url}") - - @staticmethod - def __add_header(cms_object: dict, value_type: str, value_name: str, value: str) -> None: - if value_type not in cms_object: - cms_object[value_type] = dict() - type_node = cms_object[value_type] - if value_name in type_node: - name_node = type_node[value_name] - if isinstance(name_node, list): - name_node.append(value) - else: - type_node[value_name] = [name_node, value] - else: - type_node[value_name] = value + raise Exception("Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) \ No newline at end of file diff --git a/bclib/listener/message.py b/bclib/listener/message.py index 72ae779..4a488fd 100644 --- a/bclib/listener/message.py +++ b/bclib/listener/message.py @@ -1,20 +1,25 @@ import asyncio import json from typing import Any -from bclib.listener.message_type import MessageType from abc import abstractmethod +from bclib.listener.message_type import MessageType + class Message: - def __init__(self, sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - self.session_id = sessionId - self.type = messageType + def __init__(self, session_id: str, message_type: 'MessageType', buffer: bytes = None) -> None: + self.session_id = session_id + self.type = message_type self.buffer = buffer - @abstractmethod - def create_response_message(self, session_id: str, buffer: bytes) -> "Message": - return Message.create_add_hock(session_id,buffer) + async def fill_async(self): + pass + + async def get_json_async(self): + return json.loads(self.buffer) + async def set_result_async(self, result: dict): + raise NotImplementedError("{0}::set_result_async",self.__class__.__name__) async def write_to_stream_async(self, stream: asyncio.StreamWriter) -> bool: is_send = True @@ -67,3 +72,13 @@ def create(session_id: str, data: Any): ret_val = Message.create_from_object( session_id, data) return ret_val + + +class ByteArrayMessage(Message): + def __init__(self, session_id: str, message_type: 'MessageType', buffer: bytes): + super().__init__(session_id, message_type, buffer) + + +class JsonBaseMessage(Message): + def __init__(self, session_id: str, message_type: 'MessageType'): + super().__init__(session_id, message_type, None) diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index 2d01e09..dfa6218 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -1,16 +1,16 @@ from struct import error from bclib.context import RabbitContext from typing import TYPE_CHECKING -from .rabbit_listener import RabbitListener +from bclib.listener.rabbit_listener import RabbitListener from bclib.utility import DictEx if TYPE_CHECKING: - from bclib import dispatcher + from bclib.dispatcher import IDispatcher class RabbitBusListener(RabbitListener): - def __init__(self, rabbit_options: DictEx, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, rabbit_options: 'DictEx', dispatcher: 'IDispatcher') -> None: super().__init__(rabbit_options) self.__dispatcher = dispatcher diff --git a/bclib/listener/rabbit_listener.py b/bclib/listener/rabbit_listener.py index 5b698d9..a8443da 100644 --- a/bclib/listener/rabbit_listener.py +++ b/bclib/listener/rabbit_listener.py @@ -1,7 +1,7 @@ import asyncio from abc import ABC, abstractmethod -from bclib.utility import DictEx import pika +from bclib.utility import DictEx class RabbitListener(ABC): def __init__(self, connection_options: DictEx) -> None: diff --git a/bclib/listener/receive_message.py b/bclib/listener/receive_message.py index 7a42c90..45643fe 100644 --- a/bclib/listener/receive_message.py +++ b/bclib/listener/receive_message.py @@ -1,9 +1,66 @@ import asyncio +import json -from .message_type import MessageType +from bclib.listener.message_type import MessageType from .message import Message +class SocketMessage(Message): + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + self.reader = reader + self.writer = writer + + async def fill_async(self): + data = await self.reader.readexactly(1) + if data: + self.type = MessageType(int.from_bytes( + data, byteorder='big', signed=True)) + data = await self.reader.readexactly(4) + data_len = int.from_bytes(data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.session_id = data.decode("utf-8") + if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): + data = await self.reader.readexactly(4) + data_len = int.from_bytes( + data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.buffer = data + + async def write_result_async(self, cms: dict,message_type:'MessageType'): + try: + self.writer.write(message_type.value.to_bytes(1, 'big')) + data = self.session_id.encode() + data_length_bytes = len(data).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(data) + if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): + result_bytes = json.dumps( + cms, ensure_ascii=False).encode("utf-8") + data_length_bytes = len(result_bytes).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(result_bytes) + await self.writer.drain() + except asyncio.CancelledError: + pass + + async def set_result_async(self, cms: dict): + await self.write_result_async(cms,self.type) + + async def read_next_message_async(self) -> 'SocketMessage': + ret_val = SocketMessage(self.reader, self.writer) + await ret_val.fill_async() + return ret_val + +# class EndPointMessage(StreamBaseMessage): +# def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): +# super().__init__(reader, writer) + + +# class SocketMessage(StreamBaseMessage): +# def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): +# super().__init__(reader, writer) + + class ReceiveMessage(Message): def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: super().__init__(sessionId, messageType, buffer) diff --git a/bclib/listener/socket_listener.py b/bclib/listener/socket_listener.py index 42746bb..c4c586b 100644 --- a/bclib/listener/socket_listener.py +++ b/bclib/listener/socket_listener.py @@ -1,7 +1,9 @@ import asyncio from typing import Callable, Coroutine -from ..listener.message import Message -from ..listener.endpoint import Endpoint + +from bclib.listener.receive_message import SocketMessage +from bclib.listener.message import Message +from bclib.listener.endpoint import Endpoint class SocketListener: @@ -13,11 +15,6 @@ def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call self.__receiver_server: asyncio.AbstractServer = None self.__sender_server: asyncio.AbstractServer = None - async def __process_message_async(self, message: 'Message') -> None: - result = await self.on_message_receive(message) - if result: - await self.send_message_async(result) - async def send_message_async(self, message: Message) -> bool: try: await message.write_to_stream_async(self.__sender_stream_writer) @@ -60,9 +57,10 @@ async def on_receiver_client_connect(self, reader: asyncio.StreamReader, writer: cause = "closed!" try: while True: - message = await Message.read_from_stream_async(reader) + # message = await Message.read_from_stream_async(reader) + message = SocketMessage(reader, writer) if message: - loop.create_task(self.__process_message_async(message)) + loop.create_task(self.on_message_receive(message)) except asyncio.CancelledError: cause = 'closed by receiver!' except (ConnectionResetError, asyncio.IncompleteReadError): diff --git a/bclib/listener/web_message.py b/bclib/listener/web_message.py index 979de53..dea36a9 100644 --- a/bclib/listener/web_message.py +++ b/bclib/listener/web_message.py @@ -1,20 +1,189 @@ -from typing import Any, Coroutine, Optional, Union +import base64 +import pathlib +import uuid +import cgi +import io +import datetime from aiohttp import web +from urllib.parse import unquote, parse_qs +from typing import Any, Coroutine, Optional, Union from aiohttp.web_response import ContentCoding +from multidict import MultiDict +from utility.response_types import ResponseTypes +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.message import Message from bclib.listener.message_type import MessageType class WebMessage(Message): - def __init__(self, request: 'web.Request',sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - super().__init__(sessionId,messageType,buffer) + _id = 0 + def __init__(self, request: 'web.Request') -> None: + super().__init__(session_id=str(uuid.uuid4()),message_type=MessageType.AD_HOC,buffer=None) self.__request = request - self.Response = None + self.Response: web.Response = None + + async def get_json_async(self) -> Coroutine[Any, Any, dict]: + return await self.create_cms_async(self.__request) + + async def set_result_async(self,cms:dict): + cms_cms = cms[HttpBaseDataType.CMS] + cms_cms_webserver = cms_cms[HttpBaseDataType.WEB_SERVER] + index = cms_cms_webserver[HttpBaseDataName.INDEX] + header_code: str = cms_cms_webserver[HttpBaseDataName.HEADER_CODE] + mime = cms_cms[HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] + headers = MultiDict() + if HttpBaseDataName.HTTP in cms_cms: + http: dict = cms_cms[HttpBaseDataName.HTTP] + for key, value in http.items(): + if isinstance(value, list): + for item in value: + headers.add(key, item) + else: + headers.add(key, value) + headers.add("Content-Type", mime) + if index == ResponseTypes.STATIC_FILE: + try: + path = pathlib.Path(cms_cms_webserver[HttpBaseDataName.FILE_PATH]) + path.stat() + self.Response = web.FileResponse( + path=path, + chunk_size=256*1024, + status=int(header_code.split(' ')[0]), + headers=headers + ) + except FileNotFoundError: + self.Response = web.Response( + status=404, + reason="File not found" + ) + else: + self.Response = web.Response( + status=int(header_code.split(' ')[0]), + headers=headers + ) + if HttpBaseDataName.CONTENT in cms_cms: + self.Response.text = cms_cms[HttpBaseDataName.CONTENT] + else: + raw_blob_content = cms_cms[HttpBaseDataName.BLOB_CONTENT] + #TODO:Check for remove extra encoding + self.Response.body = base64.b64decode(raw_blob_content.encode("utf-8")) + + @staticmethod + async def create_cms_async(request: 'web.Request') -> dict: + cms_object = dict() + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.METHODE, request.method.lower()) + raw_url = unquote(request.path_qs)[1:] + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.RAW_URL, raw_url) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.URL, request.path[1:]) + WebMessage.__add_query_string(request.query, cms_object) + for key, value in request.headers.items(): + field_name = key.strip().lower() + if field_name == HttpBaseDataName.COOKIE: + WebMessage.__add_cookie(value, cms_object) + elif field_name == HttpBaseDataName.HOST: + WebMessage.__add_host(value, raw_url, cms_object) + else: + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + field_name, str(value).strip()) + WebMessage.__add_server_data(cms_object, request) + await WebMessage.__add_body_async(cms_object, request) + return {"cms": cms_object} + + @staticmethod + async def __add_body_async(cms_object: dict, request: 'web.Request'): + content_len_str = request.headers.get('Content-Length') + if content_len_str or request.can_read_body: + raw_body = await request.read() + body = raw_body.decode('utf-8') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.BODY, body) + content_type: str = request.headers.get( + "content-type") + if content_type and content_type.find("application/json") < 0: + if content_type.find("multipart/form-data") >= 0: + _, content_type_value_params = cgi.parse_header( + content_type) + content_type_value_params['boundary'] = bytes( + content_type_value_params['boundary'], "utf-8") + if content_len_str is not None: + content_type_value_params['CONTENT-LENGTH'] = int( + content_len_str) + with io.BytesIO(raw_body) as stream: + fields = cgi.parse_multipart( + stream, content_type_value_params) + for key, value in fields.items(): + WebMessage.__add_header(cms_object, + HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) + else: + for key, value in parse_qs(body).items(): + WebMessage.__add_header( + cms_object, HttpBaseDataType.FORM, key, value[0] if len(value) == 1 else value) + + @staticmethod + def __add_server_data(cms_object: dict, request: 'web.Request'): + WebMessage._id += 1 + now = datetime.datetime.now() + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE, now.strftime("%d/%m/%Y")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.TIME, now.strftime("%H:%M")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE2, now.strftime("%Y%m%d")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.TIME2, now.strftime("%H%M%S")) + WebMessage.__add_header(cms_object, HttpBaseDataType.CMS, + HttpBaseDataName.DATE3, now.strftime("%Y.%m.%d")) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.REQUEST_ID, str(WebMessage._id)) + host_parts = request.host.split(':') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST_IP, host_parts[0]) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST_PORT, host_parts[1] if len(host_parts) > 1 else "80") # edit + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.CLIENT_IP, str(request.remote)) + + @staticmethod + def __add_query_string(query: dict, cms_object: dict) -> None: + for key, value in query.items(): + WebMessage.__add_header( + cms_object, HttpBaseDataType.QUERY, key, value) + + @staticmethod + def __add_cookie(raw_header_value: str, cms_object) -> None: + for item in raw_header_value.split(';'): + parts = item.split('=') + if len(parts) == 2: + WebMessage.__add_header(cms_object, HttpBaseDataName.COOKIE, + parts[0].strip(), parts[1].strip()) + @staticmethod + def __add_host(row_host: str, row_url: str, cms_object) -> None: + host_parts = row_host.split(':') + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.HOST, host_parts[0]) + if len(host_parts) == 2: + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.PORT, host_parts[1]) + WebMessage.__add_header(cms_object, HttpBaseDataType.REQUEST, + HttpBaseDataName.FULL_URL, f"{row_host}/{row_url}") - def create_response_message(self, session_id: str, buffer: bytes) -> "Message": - ret_val = WebMessage (self.__request, session_id,MessageType.AD_HOC,buffer) - ret_val.Response = self.Response - return ret_val + @staticmethod + def __add_header(cms_object: dict, value_type: str, value_name: str, value: str) -> None: + if value_type not in cms_object: + cms_object[value_type] = dict() + type_node = cms_object[value_type] + if value_name in type_node: + name_node = type_node[value_name] + if isinstance(name_node, list): + name_node.append(value) + else: + type_node[value_name] = [name_node, value] + else: + type_node[value_name] = value async def start_stream_response_async(self,status: int = 200, reason: Optional[str] = 'OK', diff --git a/bclib/logger/ilogger.py b/bclib/logger/ilogger.py index 13764b7..b1526e8 100644 --- a/bclib/logger/ilogger.py +++ b/bclib/logger/ilogger.py @@ -29,9 +29,9 @@ def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, ** def log_request(self, message:'str'): if(self.__log_request): - print(self.__log_name, message) + print(self.__log_name,'LOG', message) def log_error(self,error:'Exception'): - print(self.__log_name,str(error)) + print(self.__log_name,'ERROR',str(error)) traceback.print_exc() diff --git a/test/dev_server/simple.py b/test/dev_server/simple.py index bec1ad8..0f693d7 100644 --- a/test/dev_server/simple.py +++ b/test/dev_server/simple.py @@ -3,12 +3,15 @@ from bclib import edge from bclib.edge import EdgeContainer from dependency_injector import containers, providers -from dependency_injector.wiring import Provide, inject +from dependency_injector.wiring import Provide, inject + def manage_resource1(): print("init tasks 1") yield print("stop tasks 1") + + async def manage_resource2_async(): await asyncio.sleep(.5) print("init tasks 2") @@ -16,36 +19,41 @@ async def manage_resource2_async(): await asyncio.sleep(.5) print("stop tasks 2") + @containers.copy(EdgeContainer) class AppContainer(EdgeContainer): app_resource1 = providers.Resource(manage_resource1) - #app_resource2 = providers.Resource(manage_resource2_async) - object_provider = providers.Callable(lambda: random.randint(100,999)) - object_val = providers.Callable(lambda: random.randint(100,999)) - object_val2 = providers.Callable(lambda: random.randint(1000,9999)) + # app_resource2 = providers.Resource(manage_resource2_async) + object_provider = providers.Callable(lambda: random.randint(100, 999)) + object_val = providers.Callable(lambda: random.randint(100, 999)) + object_val2 = providers.Callable(lambda: random.randint(1000, 9999)) -container=AppContainer() + +container = AppContainer() container.app_config.from_dict({ - "server": "localhost:8080", - "router": "web", - #"loop":asyncio.get_event_loop() - "cache":{"1":22} - }) + "server": "localhost:8080", + "router": "web", + # "loop":asyncio.get_event_loop() + "cache": {"1": 22} +}) + +app = edge.from_container(container) -app = edge.create_server(container) async def check_async(context: edge.RequestContext): return context.url.endswith("app") + @app.web_action(app.callback(check_async)) def process_web_action(context: edge.WebContext): return "result from process_web_action" + @app.web_action() @inject -def process_default_web_action(context: edge.WebContext,val=Provide["object_provider"],container:AppContainer =Provide["app_container"]): - return "result from process_default_web_action " + str(val or "?") +" "+ str(context.dispatcher.container.object_val()) + " "+ str(container.object_val2()) +def process_default_web_action(context: edge.WebContext, val=Provide["object_provider"], container: AppContainer = Provide["app_container"]): + return "result from process_default_web_action " + str(val or "?") + " " + str(context.dispatcher.container.object_val()) + " " + str(container.object_val2()) container.wire(modules=[__name__]) diff --git a/test/web/simple.py b/test/web/simple.py index 170618e..8e5e552 100644 --- a/test/web/simple.py +++ b/test/web/simple.py @@ -1,9 +1,11 @@ import asyncio from concurrent.futures import thread -from bclib import edge +# from bclib import edge import time +from bclib import edge + options = { "endpoint": "127.0.0.1:1025", @@ -12,8 +14,6 @@ app = edge.from_options(options) -print(app) - @app.web_action() async def process_web_remain_request(context: edge.WebContext): diff --git a/test/web_socket/list-data.py b/test/web_socket/list-data.py index 6ea9cac..b71f328 100644 --- a/test/web_socket/list-data.py +++ b/test/web_socket/list-data.py @@ -82,15 +82,15 @@ async def process_message_async(context: edge.SocketContext): f'message of type {msg.type} come from {msg.session_id} in {datetime.datetime.now()}') future = context.dispatcher.run_in_background( send_data_async, context) - while True: + while not future.done(): try: msg = await context.read_message_async() print( f'message of type {msg.type} come from {msg.session_id} in {datetime.datetime.now()}') if msg.type in [edge.MessageType.DISCONNECT, edge.MessageType.NOT_EXIST]: break - except: - print("connection closed!") + except Exception as ex: + print("connection closed!",ex) break future.cancel() From fda4d25a7c846860f6e61cb357fe222dbe1351a6 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Wed, 4 Dec 2024 23:36:33 +0330 Subject: [PATCH 06/15] clean code --- bclib/context/context_factory.py | 68 +------------- bclib/context/socket_context.py | 6 -- bclib/dispatcher/endpoint_dispatcher.py | 4 +- bclib/dispatcher/idispatcher.py | 4 - bclib/dispatcher/routing_dispatcher.py | 6 +- bclib/dispatcher/socket_dispatcher.py | 9 +- bclib/listener/__init__.py | 3 +- bclib/listener/http_listener/http_listener.py | 12 +-- bclib/listener/message.py | 85 ++--------------- bclib/listener/receive_message.py | 93 ------------------- bclib/listener/socket_listener.py | 13 +-- bclib/listener/socket_message.py | 59 ++++++++++++ bclib/listener/web_message.py | 4 +- 13 files changed, 83 insertions(+), 283 deletions(-) delete mode 100644 bclib/listener/receive_message.py create mode 100644 bclib/listener/socket_message.py diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py index b00cd1a..5cbd4d8 100644 --- a/bclib/context/context_factory.py +++ b/bclib/context/context_factory.py @@ -1,7 +1,6 @@ -import json import re from struct import error -from typing import Optional, TYPE_CHECKING +from typing import Callable, Optional, TYPE_CHECKING if TYPE_CHECKING: @@ -9,16 +8,6 @@ from bclib.utility import DictEx from bclib.logger import ILogger from . import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext - - -# from .client_source_context import ClientSourceContext -# from .restful_context import RESTfulContext -# from .web_context import WebContext -# from .request_context import RequestContext -# from .context import Context -# from .socket_context import SocketContext -# from .server_source_context import ServerSourceContext -# from .named_pipe_context import NamedPipeContext from bclib.listener.message import Message, MessageType from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType @@ -131,57 +120,4 @@ async def create_context_async(self, message: 'Message', dispatcher: 'IDispatche else: raise NameError( f"Configured context type '{context_type}' not found for '{url}'") - return ret_val - - # async def create_context_async(self, message: 'Message', dispatcher:'IDispatcher') -> Context: - # """Create context from message object""" - - # print('qam',json.dumps( message)) - # ret_val: RequestContext = None - # context_type = None - # cms_object: Optional[dict] = None - # url: Optional[str] = None - # request_id: Optional[str] = None - # method: Optional[str] = None - # message_json: Optional[dict] = None - # if True: #message.buffer is not None: - # #message_json = json.loads(message.buffer) - # message_json = await message.get_json_async() - # cms_object = message_json[HttpBaseDataType.CMS] if HttpBaseDataType.CMS in message_json else None - # if cms_object: - # if 'request' in cms_object: - # req = cms_object["request"] - # else: - # raise KeyError("request key not found in cms object") - # if 'full-url' in req: - # url = req["full-url"] - # else: - # raise KeyError("full-url key not found in request") - # request_id = req['request-id'] if 'request-id' in req else 'none' - # method = req['methode'] if 'methode' in req else 'none' - # if message.type == MessageType.AD_HOC: - # if url or self.__default_router is None: - # context_type = self.__context_type_detector(url) - # else: - # context_type = self.__default_router - # else: - # context_type = "socket" - # self.logger.log_request( - # f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") - - # if context_type == "client_source": - # ret_val = ClientSourceContext(cms_object, dispatcher,message) - # elif context_type == "restful": - # ret_val = RESTfulContext(cms_object, dispatcher,message) - # elif context_type == "server_source": - # ret_val = ServerSourceContext(message_json, dispatcher) - # elif context_type == "web": - # ret_val = WebContext(cms_object, dispatcher,message) - # elif context_type == "socket": - # ret_val = SocketContext(cms_object, dispatcher, message, message_json) - # elif context_type is None: - # raise NameError(f"No context found for '{url}'") - # else: - # raise NameError( - # f"Configured context type '{context_type}' not found for '{url}'") - # return ret_val + return ret_val \ No newline at end of file diff --git a/bclib/context/socket_context.py b/bclib/context/socket_context.py index 5f0ba72..833acf0 100644 --- a/bclib/context/socket_context.py +++ b/bclib/context/socket_context.py @@ -3,7 +3,6 @@ from bclib.listener.message_type import MessageType from bclib.context import Context -from bclib.listener.receive_message import ReceiveMessage from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, HttpStatusCodes, ResponseTypes if TYPE_CHECKING: @@ -41,14 +40,9 @@ async def send_content_async(self, cms = Context._generate_response_cms( content, response_type, status_code, mime, template, headers) await self.message.write_result_async(cms,MessageType.MESSAGE) - # message = ReceiveMessage.create_from_object( - # self.message.session_id, cms) - # await message.write_to_stream_async(self.message.writer) async def read_message_async(self) -> 'listener.SocketMessage': return await self.message.read_next_message_async() async def send_close_async(self) -> None: - #message = ReceiveMessage.create_disconnect(self.message.session_id) - #await message.write_to_stream_async(self.message.writer) await self.message.write_result_async(None,MessageType.DISCONNECT) diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index 2f4a5fa..741a038 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -2,13 +2,13 @@ from context.context_factory import ContextFactory from dependency_injector import containers -from listener.receive_message import SocketMessage +from listener.socket_message import SocketMessage from bclib.cache import CacheManager from bclib.db_manager import DbManager from bclib.logger import ILogger from bclib.utility import DictEx -from bclib.listener import Endpoint, ReceiveMessage +from bclib.listener import Endpoint from .routing_dispatcher import RoutingDispatcher diff --git a/bclib/dispatcher/idispatcher.py b/bclib/dispatcher/idispatcher.py index eb69fe8..2d50a07 100644 --- a/bclib/dispatcher/idispatcher.py +++ b/bclib/dispatcher/idispatcher.py @@ -56,10 +56,6 @@ async def dispatch_async(self, context: 'Context') -> Any: def dispatch_in_background(self, context: 'Context') -> asyncio.Future: """Dispatch context in background""" - @abstractmethod - async def send_message_async(self, message: Message) -> None: - """Send message to endpoint""" - def run_in_background(self, callback: Callable, *args: Any) -> asyncio.Future: """helper for run function in background thread""" diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index 1296915..29ab5f6 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -1,6 +1,5 @@ import asyncio import inspect -import json from typing import Callable, Any, Coroutine, Optional from dependency_injector import containers @@ -10,7 +9,7 @@ from bclib.cache import CacheManager from bclib.utility import DictEx from bclib.context.context_factory import ContextFactory -from bclib.listener import Message, MessageType, HttpBaseDataType, ReceiveMessage +from bclib.listener import Message, MessageType from .dispatcher_helper import DispatcherHelper from .dispatcher import Dispatcher @@ -22,11 +21,10 @@ def __init__(self, container: 'containers.Container', context_factory: 'ContextF cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self._context_factory = context_factory - async def _on_message_receive_async(self, message: Message): + async def _on_message_receive_async(self, message: Message) -> Coroutine: """Process received message""" try: - await message.fill_async() context = await self._context_factory.create_context_async(message, self) response = await self.dispatch_async(context) await message.set_result_async(response) diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index dd0d205..a4ae832 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -6,25 +6,18 @@ from bclib.db_manager import DbManager from bclib.logger import ILogger from bclib.utility import DictEx -from bclib.listener import Endpoint, Message, SocketListener +from bclib.listener import Endpoint, SocketListener from .routing_dispatcher import RoutingDispatcher class SocketDispatcher(RoutingDispatcher): def __init__(self, container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): super().__init__(container= container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) - self.__lock = asyncio.Lock() self.__listener = SocketListener( Endpoint(self.options.receiver), Endpoint(self.options.sender), self._on_message_receive_async) - async def send_message_async(self, message: Message) -> bool: - """Send message to endpoint""" - - async with self.__lock: - return await self.__listener.send_message_async(message) - def initialize_task(self): super().initialize_task() self.__listener.initialize_task(self.event_loop) diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index 8b21f36..6bf261c 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -2,10 +2,9 @@ from bclib.listener.socket_listener import SocketListener from bclib.listener.rabbit_bus_listener import RabbitBusListener from bclib.listener.message import Message -from bclib.listener.receive_message import ReceiveMessage from bclib.listener.message_type import MessageType from bclib.listener.http_listener.http_listener import HttpListener from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.web_message import WebMessage -from bclib.listener.receive_message import SocketMessage +from bclib.listener.socket_message import SocketMessage diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 816a655..327c8c5 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -1,14 +1,9 @@ import asyncio -import json import ssl -import base64 -from typing import Callable, TYPE_CHECKING, Optional, Awaitable +from typing import Callable, TYPE_CHECKING, Coroutine, Optional from ..endpoint import Endpoint -from .http_base_data_name import HttpBaseDataName -from .http_base_data_type import HttpBaseDataType -from bclib.utility import DictEx, ResponseTypes +from bclib.utility import DictEx from ..web_message import WebMessage -import pathlib if TYPE_CHECKING: from aiohttp import web @@ -30,7 +25,7 @@ class HttpListener: _DEFAULT_CLIENT_MAX_SIZE = 1024 ** 2 - def __init__(self, endpoint: Endpoint, async_callback: 'Callable[[WebMessage], Awaitable[WebMessage]]',ssl_options:'dict', configuration: Optional[DictEx]): + def __init__(self, endpoint: Endpoint, async_callback: 'Callable[[WebMessage],Coroutine]',ssl_options:'dict', configuration: Optional[DictEx]): self.__endpoint = endpoint self.on_message_receive_async = async_callback self.ssl_options = ssl_options @@ -47,7 +42,6 @@ def initialize_task(self, event_loop: asyncio.AbstractEventLoop): async def __server_task(self, event_loop: asyncio.AbstractEventLoop): from aiohttp import web - from multidict import MultiDict async def on_request_receive_async(request: 'web.Request') -> web.Response: msg = WebMessage(request) await self.on_message_receive_async(msg) diff --git a/bclib/listener/message.py b/bclib/listener/message.py index 4a488fd..8a9477b 100644 --- a/bclib/listener/message.py +++ b/bclib/listener/message.py @@ -1,84 +1,19 @@ -import asyncio -import json -from typing import Any +from typing import Any, Coroutine from abc import abstractmethod from bclib.listener.message_type import MessageType class Message: - def __init__(self, session_id: str, message_type: 'MessageType', buffer: bytes = None) -> None: - self.session_id = session_id - self.type = message_type - self.buffer = buffer - - async def fill_async(self): + def __init__(self ) -> None: + self.session_id: str = "not-set" + self.type: MessageType = MessageType.AD_HOC + + @abstractmethod + async def get_json_async(self)-> Coroutine[Any, Any, dict]: pass - async def get_json_async(self): - return json.loads(self.buffer) - - async def set_result_async(self, result: dict): - raise NotImplementedError("{0}::set_result_async",self.__class__.__name__) - - async def write_to_stream_async(self, stream: asyncio.StreamWriter) -> bool: - is_send = True - try: - stream.write(self.type.value.to_bytes(1, 'big')) - data = self.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - stream.write(data_length_bytes) - stream.write(data) - - if self.type in (MessageType.AD_HOC, MessageType.MESSAGE): - data_length_bytes = len(self.buffer).to_bytes(4, 'big') - stream.write(data_length_bytes) - stream.write(self.buffer) - await stream.drain() - except asyncio.CancelledError: - is_send = False - return is_send - - @staticmethod - def create_add_hock(session_id: str, buffer: bytes): - return Message(session_id, MessageType.AD_HOC, buffer) - - @staticmethod - def create_disconnect(session_id: str): - return Message(session_id, MessageType.DISCONNECT, None) - - @staticmethod - def create_from_text(session_id: str, text: str): - return Message(session_id, MessageType.MESSAGE, text.encode()) - - @staticmethod - def create_from_byte(session_id: str, array: bytes): - return Message(session_id, MessageType.MESSAGE, array) - - @staticmethod - def create_from_object(session_id: str, object_data: Any): - return Message(session_id, MessageType.MESSAGE, json.dumps(object_data).encode("utf-8")) - - @staticmethod - def create(session_id: str, data: Any): - ret_val: Message = None - if isinstance(data, str): - ret_val = Message.create_from_text( - session_id, data) - elif isinstance(data, bytes): - ret_val = Message.create_from_byte( - session_id, data) - else: - ret_val = Message.create_from_object( - session_id, data) - return ret_val - - -class ByteArrayMessage(Message): - def __init__(self, session_id: str, message_type: 'MessageType', buffer: bytes): - super().__init__(session_id, message_type, buffer) - + @abstractmethod + async def set_result_async(self, result: dict)-> Coroutine[Any, Any, None]: + pass -class JsonBaseMessage(Message): - def __init__(self, session_id: str, message_type: 'MessageType'): - super().__init__(session_id, message_type, None) diff --git a/bclib/listener/receive_message.py b/bclib/listener/receive_message.py deleted file mode 100644 index 45643fe..0000000 --- a/bclib/listener/receive_message.py +++ /dev/null @@ -1,93 +0,0 @@ -import asyncio -import json - -from bclib.listener.message_type import MessageType -from .message import Message - - -class SocketMessage(Message): - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): - self.reader = reader - self.writer = writer - - async def fill_async(self): - data = await self.reader.readexactly(1) - if data: - self.type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = await self.reader.readexactly(4) - data_len = int.from_bytes(data, byteorder='big', signed=True) - data = await self.reader.readexactly(data_len) - self.session_id = data.decode("utf-8") - if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = await self.reader.readexactly(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = await self.reader.readexactly(data_len) - self.buffer = data - - async def write_result_async(self, cms: dict,message_type:'MessageType'): - try: - self.writer.write(message_type.value.to_bytes(1, 'big')) - data = self.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - self.writer.write(data_length_bytes) - self.writer.write(data) - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): - result_bytes = json.dumps( - cms, ensure_ascii=False).encode("utf-8") - data_length_bytes = len(result_bytes).to_bytes(4, 'big') - self.writer.write(data_length_bytes) - self.writer.write(result_bytes) - await self.writer.drain() - except asyncio.CancelledError: - pass - - async def set_result_async(self, cms: dict): - await self.write_result_async(cms,self.type) - - async def read_next_message_async(self) -> 'SocketMessage': - ret_val = SocketMessage(self.reader, self.writer) - await ret_val.fill_async() - return ret_val - -# class EndPointMessage(StreamBaseMessage): -# def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): -# super().__init__(reader, writer) - - -# class SocketMessage(StreamBaseMessage): -# def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): -# super().__init__(reader, writer) - - -class ReceiveMessage(Message): - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, sessionId: str, messageType: MessageType, buffer: bytes = None) -> None: - super().__init__(sessionId, messageType, buffer) - self.reader = reader - self.writer = writer - - async def read_next_message_async(self) -> 'ReceiveMessage': - return await ReceiveMessage.read_from_stream_async(self.reader, self.writer) - - @staticmethod - async def read_from_stream_async(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> 'ReceiveMessage': - message: ReceiveMessage = None - data = await reader.readexactly(1) - if data: - message_type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = await reader.readexactly(4) - data_len = int.from_bytes(data, byteorder='big', signed=True) - data = await reader.readexactly(data_len) - session_id = data.decode("utf-8") - parameter = None - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = await reader.readexactly(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = await reader.readexactly(data_len) - parameter = data - message = ReceiveMessage( - reader, writer, session_id, message_type, parameter) - return message diff --git a/bclib/listener/socket_listener.py b/bclib/listener/socket_listener.py index c4c586b..a14b797 100644 --- a/bclib/listener/socket_listener.py +++ b/bclib/listener/socket_listener.py @@ -1,13 +1,12 @@ import asyncio from typing import Callable, Coroutine -from bclib.listener.receive_message import SocketMessage -from bclib.listener.message import Message +from bclib.listener.socket_message import SocketMessage from bclib.listener.endpoint import Endpoint class SocketListener: - def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call_back: 'Callable[[Message], Coroutine[Message]]'): + def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call_back: 'Callable[[SocketMessage],Coroutine]'): self.__receiver_endpoint = receiver self.__sender_endpoint = sender self.on_message_receive = on_message_receive_call_back @@ -15,13 +14,6 @@ def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call self.__receiver_server: asyncio.AbstractServer = None self.__sender_server: asyncio.AbstractServer = None - async def send_message_async(self, message: Message) -> bool: - try: - await message.write_to_stream_async(self.__sender_stream_writer) - except Exception as ex: - print(f"Error in send message {ex}") - return False - async def on_sender_client_connect(self, _: asyncio.StreamReader, writer: asyncio.StreamWriter): peer_name = writer.get_extra_info('peername') print(f'Reader from {peer_name}, connect to sender!') @@ -57,7 +49,6 @@ async def on_receiver_client_connect(self, reader: asyncio.StreamReader, writer: cause = "closed!" try: while True: - # message = await Message.read_from_stream_async(reader) message = SocketMessage(reader, writer) if message: loop.create_task(self.on_message_receive(message)) diff --git a/bclib/listener/socket_message.py b/bclib/listener/socket_message.py new file mode 100644 index 0000000..3a6fc7f --- /dev/null +++ b/bclib/listener/socket_message.py @@ -0,0 +1,59 @@ +import asyncio +import json +from typing import Any, Coroutine, Optional + +from bclib.listener.message_type import MessageType +from .message import Message + + +class SocketMessage(Message): + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + self.reader = reader + self.writer = writer + self.buffer:Optional[bytes] = None + + async def get_json_async(self)-> Coroutine[Any, Any, dict]: + await self.__fill_async() + return json.loads(self.buffer) + + async def __fill_async(self) -> Coroutine[Any, Any, None]: + if self.buffer is None: + data = await self.reader.readexactly(1) + if data: + self.type = MessageType(int.from_bytes( + data, byteorder='big', signed=True)) + data = await self.reader.readexactly(4) + data_len = int.from_bytes(data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.session_id = data.decode("utf-8") + if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): + data = await self.reader.readexactly(4) + data_len = int.from_bytes( + data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.buffer = data + + async def write_result_async(self, cms: dict,message_type:'MessageType'): + try: + self.writer.write(message_type.value.to_bytes(1, 'big')) + data = self.session_id.encode() + data_length_bytes = len(data).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(data) + if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): + result_bytes = json.dumps( + cms, ensure_ascii=False).encode("utf-8") + data_length_bytes = len(result_bytes).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(result_bytes) + await self.writer.drain() + except asyncio.CancelledError: + pass + + async def set_result_async(self, cms: dict): + await self.write_result_async(cms,self.type) + + async def read_next_message_async(self) -> 'SocketMessage': + ret_val = SocketMessage(self.reader, self.writer) + await ret_val.__fill_async() + return ret_val \ No newline at end of file diff --git a/bclib/listener/web_message.py b/bclib/listener/web_message.py index dea36a9..6abca31 100644 --- a/bclib/listener/web_message.py +++ b/bclib/listener/web_message.py @@ -1,6 +1,5 @@ import base64 import pathlib -import uuid import cgi import io import datetime @@ -13,12 +12,11 @@ from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.message import Message -from bclib.listener.message_type import MessageType class WebMessage(Message): _id = 0 def __init__(self, request: 'web.Request') -> None: - super().__init__(session_id=str(uuid.uuid4()),message_type=MessageType.AD_HOC,buffer=None) + super().__init__() self.__request = request self.Response: web.Response = None From a7c0a6f895183e1abcee8f230d79a4830797f210 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Sun, 8 Dec 2024 13:59:57 +0330 Subject: [PATCH 07/15] fix reported issue --- bclib/context/context_factory.py | 6 +-- bclib/dispatcher/dev_server_dispatcher.py | 2 +- bclib/dispatcher/endpoint_dispatcher.py | 7 ++-- bclib/dispatcher/socket_dispatcher.py | 4 +- bclib/edge.py | 41 ++----------------- bclib/listener/web_message.py | 4 +- .../exchange_rabbit_schema_base_logger.py | 2 +- bclib/logger/logger_factory.py | 4 +- 8 files changed, 18 insertions(+), 52 deletions(-) diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py index 5cbd4d8..afe6b6e 100644 --- a/bclib/context/context_factory.py +++ b/bclib/context/context_factory.py @@ -4,9 +4,9 @@ if TYPE_CHECKING: - from dispatcher.idispatcher import IDispatcher - from bclib.utility import DictEx + from bclib.dispatcher.idispatcher import IDispatcher from bclib.logger import ILogger +from bclib.utility import DictEx from . import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext from bclib.listener.message import Message, MessageType from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType @@ -38,7 +38,7 @@ def __init_router_lookup(self, router: 'DictEx'): route_dict = dict() for key, values in router.items(): - k = k.strip() + key = key.strip() if key != 'rabbit': if '*' in values: route_dict['*'] = key diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index b586999..96e7680 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -8,7 +8,7 @@ from bclib.db_manager import DbManager from bclib.utility import DictEx from bclib.listener import Endpoint, HttpListener -from .routing_dispatcher import RoutingDispatcher +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class DevServerDispatcher(RoutingDispatcher): diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index 741a038..c398975 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -1,15 +1,14 @@ import asyncio -from context.context_factory import ContextFactory from dependency_injector import containers -from listener.socket_message import SocketMessage +from bclib.context.context_factory import ContextFactory +from bclib.listener.socket_message import SocketMessage from bclib.cache import CacheManager from bclib.db_manager import DbManager from bclib.logger import ILogger - from bclib.utility import DictEx from bclib.listener import Endpoint -from .routing_dispatcher import RoutingDispatcher +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class EndpointDispatcher(RoutingDispatcher): diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index a4ae832..fad18d4 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -1,13 +1,13 @@ import asyncio -from context.context_factory import ContextFactory from dependency_injector import containers +from bclib.context.context_factory import ContextFactory from bclib.cache import CacheManager from bclib.db_manager import DbManager from bclib.logger import ILogger from bclib.utility import DictEx from bclib.listener import Endpoint, SocketListener -from .routing_dispatcher import RoutingDispatcher +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class SocketDispatcher(RoutingDispatcher): diff --git a/bclib/edge.py b/bclib/edge.py index 82853d3..babec2b 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -37,43 +37,10 @@ def from_list(hosts: 'dict[str,list[str]]'): args.append("-m") tasks.append(loop.run_in_executor(executor, subprocess.run, args)) print(f'{host} start running from {args[1]}') - loop.run_until_complete(asyncio.gather(*tasks)) - - - -# def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: -# """Create related RoutingDispatcher obj from config object""" - -# import sys -# import getopt - -# multi: bool = False -# argument_list = sys.argv[1:] -# # Options -# short_options = "mn:" -# # Long options -# long_options = ["Name =", "Multi"] -# try: -# arguments, _ = getopt.gnu_getopt( -# argument_list, short_options, long_options) -# for current_argument, current_value in arguments: -# if current_argument in ("-n", "--Name"): -# options["name"] = current_value.strip() -# elif current_argument in ("-m", "--Multi"): -# multi = True -# except getopt.error as err: -# print(str(err)) - -# if not multi: -# __print_splash(False) -# ret_val: RoutingDispatcher = None -# if "server" in options: -# ret_val = DevServerDispatcher(options=options,loop=loop) -# elif "endpoint" in options: -# ret_val = EndpointDispatcher(options=options,loop=loop) -# else: -# ret_val = SocketDispatcher(options=options,loop=loop) -# return ret_val + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except KeyboardInterrupt: + pass def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: container=EdgeContainer() diff --git a/bclib/listener/web_message.py b/bclib/listener/web_message.py index 6abca31..e35d3a9 100644 --- a/bclib/listener/web_message.py +++ b/bclib/listener/web_message.py @@ -6,9 +6,9 @@ from aiohttp import web from urllib.parse import unquote, parse_qs from typing import Any, Coroutine, Optional, Union -from aiohttp.web_response import ContentCoding from multidict import MultiDict -from utility.response_types import ResponseTypes +from aiohttp.web_response import ContentCoding +from bclib.utility.response_types import ResponseTypes from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.message import Message diff --git a/bclib/logger/exchange_rabbit_schema_base_logger.py b/bclib/logger/exchange_rabbit_schema_base_logger.py index f08ae0e..bcfc185 100644 --- a/bclib/logger/exchange_rabbit_schema_base_logger.py +++ b/bclib/logger/exchange_rabbit_schema_base_logger.py @@ -1,6 +1,6 @@ import asyncio import json -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx diff --git a/bclib/logger/logger_factory.py b/bclib/logger/logger_factory.py index 41a6abd..596230d 100644 --- a/bclib/logger/logger_factory.py +++ b/bclib/logger/logger_factory.py @@ -20,9 +20,9 @@ def create(options: DictEx) -> ILogger: else: logger_type = logger_option.type.lower() if logger_type == 'schema.restful': - logger = RESTfulSchemaBaseLogger(logger_option) + logger = RESTfulSchemaBaseLogger(options) elif logger_type == "schema.rabbit": - logger = RabbitSchemaBaseLogger(logger_option) + logger = RabbitSchemaBaseLogger(options) else: raise Exception( f"Type '{logger_type}' not support for logger") From 6dbf50538685dbd6c3c22bd36d3a0f6ce8ba8661 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:28:05 +0330 Subject: [PATCH 08/15] add endpoint context --- bclib/context/__init__.py | 1 + bclib/context/context_factory.py | 13 +++++- bclib/context/end_point_context.py | 47 +++++++++++++++++++ bclib/context/socket_context.py | 1 + bclib/dispatcher/dispatcher.py | 25 +++++++++- bclib/dispatcher/endpoint_dispatcher.py | 4 +- bclib/dispatcher/idispatcher.py | 1 - bclib/edge.py | 2 +- bclib/listener/__init__.py | 1 + bclib/listener/end_point_message.py | 12 +++++ bclib/listener/rabbit_bus_listener.py | 2 +- bclib/listener/socket_listener.py | 2 +- bclib/listener/socket_message.py | 62 ++----------------------- bclib/listener/stream_base_message.py | 57 +++++++++++++++++++++++ bclib/listener/web_message.py | 3 +- test/web_socket/list-data.py | 6 +-- 16 files changed, 169 insertions(+), 70 deletions(-) create mode 100644 bclib/context/end_point_context.py create mode 100644 bclib/listener/end_point_message.py create mode 100644 bclib/listener/stream_base_message.py diff --git a/bclib/context/__init__.py b/bclib/context/__init__.py index 5f46f54..9d955be 100644 --- a/bclib/context/__init__.py +++ b/bclib/context/__init__.py @@ -12,3 +12,4 @@ from bclib.context.source_context import SourceContext from bclib.context.source_member_context import SourceMemberContext from bclib.context.context_factory import ContextFactory +from bclib.context.end_point_context import EndPointContext diff --git a/bclib/context/context_factory.py b/bclib/context/context_factory.py index afe6b6e..27b422d 100644 --- a/bclib/context/context_factory.py +++ b/bclib/context/context_factory.py @@ -7,7 +7,14 @@ from bclib.dispatcher.idispatcher import IDispatcher from bclib.logger import ILogger from bclib.utility import DictEx -from . import ClientSourceContext, RESTfulContext, WebContext, RequestContext, Context, SocketContext, ServerSourceContext +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.request_context import RequestContext +from bclib.context.socket_context import SocketContext +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.end_point_context import EndPointContext from bclib.listener.message import Message, MessageType from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType @@ -100,7 +107,7 @@ async def create_context_async(self, message: 'Message', dispatcher: 'IDispatche else: context_type = self.__default_router else: - context_type = "socket" + context_type = "endpoint" self.logger.log_request( f"({context_type}::{message.type.name}){f' - {request_id} {method} {url} ' if cms_object else ''}") @@ -112,6 +119,8 @@ async def create_context_async(self, message: 'Message', dispatcher: 'IDispatche ret_val = ServerSourceContext(message_json, dispatcher) elif context_type == "web": ret_val = WebContext(cms_object, dispatcher, message) + elif context_type == "endpoint": + ret_val = EndPointContext(cms_object, dispatcher, message, message_json) elif context_type == "socket": ret_val = SocketContext( cms_object, dispatcher, message, message_json) diff --git a/bclib/context/end_point_context.py b/bclib/context/end_point_context.py new file mode 100644 index 0000000..8f31b36 --- /dev/null +++ b/bclib/context/end_point_context.py @@ -0,0 +1,47 @@ +from typing import Any +from bclib.dispatcher.idispatcher import IDispatcher +from bclib.listener.end_point_message import EndPointMessage +from bclib.context.context import Context +from bclib.listener.message_type import MessageType +from bclib.utility import DictEx, HttpMimeTypes, HttpStatusCodes, ResponseTypes + + +import json + + +class EndPointContext(Context): + """Base class for dispatching end point socket base request context""" + + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher', message_object: 'EndPointMessage', body: 'dict') -> None: + super().__init__(dispatcher) + self.cms = DictEx(cms_object) if cms_object else None + self.url: str = self.cms.request.url if cms_object else None + self.body = DictEx(body) if body else None + self.message = message_object + self.is_adhoc = False + + async def send_object_async(self, obj) -> None: + await self.send_content_async(json.dumps(obj)) + + async def send_html_async(self, html) -> None: + await self.send_content_async(html, mime=HttpMimeTypes.HTML) + + async def send_code_async(self, code) -> None: + await self.send_content_async(code, response_type=ResponseTypes.RENDERABLE, mime=HttpMimeTypes.HTML) + + async def send_content_async(self, + content: 'Any', + response_type: 'str' = ResponseTypes.RENDERED, + status_code: 'str' = HttpStatusCodes.OK, + mime: 'str' = HttpMimeTypes.JSON, + template: 'DictEx' = None, + headers: 'dict' = None) -> None: + cms = Context._generate_response_cms( + content, response_type, status_code, mime, template, headers) + await self.message.write_result_async(cms,MessageType.MESSAGE) + + async def read_message_async(self) -> 'EndPointMessage': + return await self.message.read_next_message_async() + + async def send_close_async(self) -> None: + await self.message.write_result_async(None,MessageType.DISCONNECT) \ No newline at end of file diff --git a/bclib/context/socket_context.py b/bclib/context/socket_context.py index 833acf0..ebbb9b6 100644 --- a/bclib/context/socket_context.py +++ b/bclib/context/socket_context.py @@ -46,3 +46,4 @@ async def read_message_async(self) -> 'listener.SocketMessage': async def send_close_async(self) -> None: await self.message.write_result_async(None,MessageType.DISCONNECT) + diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index 29f09e2..5303ea8 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -18,7 +18,7 @@ from .callback_info import CallbackInfo -from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext +from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext,EndPointContext class Dispatcher(ABC): """Base class for dispatching request""" @@ -37,6 +37,29 @@ def __init__(self,container:'containers.Container', options: 'DictEx',cache_man self.__rabbit_dispatcher.append( RabbitBusListener(setting, self)) + def endpoint_action(self, * predicates: (Predicate)): + """Decorator for determine end point action""" + + def _decorator(end_point_action_handler: 'Callable[[EndPointContext],bool]'): + + @wraps(end_point_action_handler) + async def non_async_wrapper(context: 'EndPointContext'): + await self.event_loop.run_in_executor(None, end_point_action_handler, context) + return True + + @wraps(end_point_action_handler) + async def async_wrapper(context: 'EndPointContext'): + await end_point_action_handler(context) + return True + + wrapper = async_wrapper if inspect.iscoroutinefunction( + end_point_action_handler) else non_async_wrapper + + self._get_context_lookup(EndPointContext.__name__)\ + .append(CallbackInfo([*predicates], wrapper)) + return end_point_action_handler + return _decorator + def socket_action(self, * predicates: (Predicate)): """Decorator for determine Socket action""" diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index c398975..b9f19fa 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -1,8 +1,8 @@ import asyncio from dependency_injector import containers +from bclib.listener.end_point_message import EndPointMessage from bclib.context.context_factory import ContextFactory -from bclib.listener.socket_message import SocketMessage from bclib.cache import CacheManager from bclib.db_manager import DbManager from bclib.logger import ILogger @@ -22,7 +22,7 @@ def initialize_task(self): async def on_connection_open(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): try: - msg = SocketMessage(reader, writer) + msg = EndPointMessage(reader, writer) await self._on_message_receive_async(message=msg) except: pass diff --git a/bclib/dispatcher/idispatcher.py b/bclib/dispatcher/idispatcher.py index 2d50a07..5c52a7e 100644 --- a/bclib/dispatcher/idispatcher.py +++ b/bclib/dispatcher/idispatcher.py @@ -6,7 +6,6 @@ from bclib.db_manager import DbManager from bclib.cache import CacheManager -from bclib.listener import Message from bclib.utility import DictEx from bclib.logger import LogObject if TYPE_CHECKING: diff --git a/bclib/edge.py b/bclib/edge.py index babec2b..77fe842 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -4,7 +4,7 @@ from dependency_injector import providers from bclib.db_manager import * from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher -from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext +from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext,EndPointContext from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index 6bf261c..63f273c 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -8,3 +8,4 @@ from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType from bclib.listener.web_message import WebMessage from bclib.listener.socket_message import SocketMessage +from bclib.listener.end_point_message import EndPointMessage diff --git a/bclib/listener/end_point_message.py b/bclib/listener/end_point_message.py new file mode 100644 index 0000000..9735c94 --- /dev/null +++ b/bclib/listener/end_point_message.py @@ -0,0 +1,12 @@ +import asyncio + +from bclib.listener.stream_base_message import StreamBaseMessage + +class EndPointMessage(StreamBaseMessage): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): + super().__init__(reader, writer) + + async def read_next_message_async(self) -> 'EndPointMessage': + ret_val = EndPointMessage(self.reader, self.writer) + await ret_val._fill_async() + return ret_val diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index dfa6218..17bb8d1 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -1,5 +1,5 @@ from struct import error -from bclib.context import RabbitContext +from bclib.context.rabbit_context import RabbitContext from typing import TYPE_CHECKING from bclib.listener.rabbit_listener import RabbitListener from bclib.utility import DictEx diff --git a/bclib/listener/socket_listener.py b/bclib/listener/socket_listener.py index a14b797..11103d0 100644 --- a/bclib/listener/socket_listener.py +++ b/bclib/listener/socket_listener.py @@ -13,7 +13,7 @@ def __init__(self, receiver: Endpoint, sender: Endpoint, on_message_receive_call self.__sender_stream_writer: asyncio.StreamWriter = None self.__receiver_server: asyncio.AbstractServer = None self.__sender_server: asyncio.AbstractServer = None - + async def on_sender_client_connect(self, _: asyncio.StreamReader, writer: asyncio.StreamWriter): peer_name = writer.get_extra_info('peername') print(f'Reader from {peer_name}, connect to sender!') diff --git a/bclib/listener/socket_message.py b/bclib/listener/socket_message.py index 3a6fc7f..f6fbd0a 100644 --- a/bclib/listener/socket_message.py +++ b/bclib/listener/socket_message.py @@ -1,59 +1,7 @@ -import asyncio -import json -from typing import Any, Coroutine, Optional - -from bclib.listener.message_type import MessageType -from .message import Message - - -class SocketMessage(Message): - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): - self.reader = reader - self.writer = writer - self.buffer:Optional[bytes] = None +from bclib.listener.stream_base_message import StreamBaseMessage - async def get_json_async(self)-> Coroutine[Any, Any, dict]: - await self.__fill_async() - return json.loads(self.buffer) - - async def __fill_async(self) -> Coroutine[Any, Any, None]: - if self.buffer is None: - data = await self.reader.readexactly(1) - if data: - self.type = MessageType(int.from_bytes( - data, byteorder='big', signed=True)) - data = await self.reader.readexactly(4) - data_len = int.from_bytes(data, byteorder='big', signed=True) - data = await self.reader.readexactly(data_len) - self.session_id = data.decode("utf-8") - if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): - data = await self.reader.readexactly(4) - data_len = int.from_bytes( - data, byteorder='big', signed=True) - data = await self.reader.readexactly(data_len) - self.buffer = data - - async def write_result_async(self, cms: dict,message_type:'MessageType'): - try: - self.writer.write(message_type.value.to_bytes(1, 'big')) - data = self.session_id.encode() - data_length_bytes = len(data).to_bytes(4, 'big') - self.writer.write(data_length_bytes) - self.writer.write(data) - if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): - result_bytes = json.dumps( - cms, ensure_ascii=False).encode("utf-8") - data_length_bytes = len(result_bytes).to_bytes(4, 'big') - self.writer.write(data_length_bytes) - self.writer.write(result_bytes) - await self.writer.drain() - except asyncio.CancelledError: - pass - - async def set_result_async(self, cms: dict): - await self.write_result_async(cms,self.type) +import asyncio - async def read_next_message_async(self) -> 'SocketMessage': - ret_val = SocketMessage(self.reader, self.writer) - await ret_val.__fill_async() - return ret_val \ No newline at end of file +class SocketMessage(StreamBaseMessage): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): + super().__init__(reader, writer) \ No newline at end of file diff --git a/bclib/listener/stream_base_message.py b/bclib/listener/stream_base_message.py new file mode 100644 index 0000000..86e04ec --- /dev/null +++ b/bclib/listener/stream_base_message.py @@ -0,0 +1,57 @@ +import asyncio +import json +from typing import Any, Coroutine, Optional + +from bclib.listener.message_type import MessageType +from .message import Message + + +class StreamBaseMessage(Message): + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + self.reader = reader + self.writer = writer + self.buffer:Optional[bytes] = None + + async def get_json_async(self)-> Coroutine[Any, Any, dict]: + await self._fill_async() + return json.loads(self.buffer) + + async def _fill_async(self) -> Coroutine[Any, Any, None]: + if self.buffer is None: + data = await self.reader.readexactly(1) + if data: + self.type = MessageType(int.from_bytes( + data, byteorder='big', signed=True)) + data = await self.reader.readexactly(4) + data_len = int.from_bytes(data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.session_id = data.decode("utf-8") + if self.type in (MessageType.AD_HOC, MessageType.MESSAGE, MessageType.CONNECT): + data = await self.reader.readexactly(4) + data_len = int.from_bytes( + data, byteorder='big', signed=True) + data = await self.reader.readexactly(data_len) + self.buffer = data + + async def write_result_async(self, cms: dict,message_type:'MessageType'): + try: + self.writer.write(message_type.value.to_bytes(1, 'big')) + data = self.session_id.encode() + data_length_bytes = len(data).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(data) + if message_type in (MessageType.AD_HOC, MessageType.MESSAGE): + result_bytes = json.dumps( + cms, ensure_ascii=False).encode("utf-8") + data_length_bytes = len(result_bytes).to_bytes(4, 'big') + self.writer.write(data_length_bytes) + self.writer.write(result_bytes) + await self.writer.drain() + except asyncio.CancelledError: + pass + + async def set_result_async(self, cms: dict): + await self.write_result_async(cms,self.type) + + + diff --git a/bclib/listener/web_message.py b/bclib/listener/web_message.py index e35d3a9..769cacd 100644 --- a/bclib/listener/web_message.py +++ b/bclib/listener/web_message.py @@ -60,7 +60,8 @@ async def set_result_async(self,cms:dict): headers=headers ) if HttpBaseDataName.CONTENT in cms_cms: - self.Response.text = cms_cms[HttpBaseDataName.CONTENT] + value = cms_cms[HttpBaseDataName.CONTENT] + self.Response.text =value if value is None or isinstance(value,str) else str(value) else: raw_blob_content = cms_cms[HttpBaseDataName.BLOB_CONTENT] #TODO:Check for remove extra encoding diff --git a/test/web_socket/list-data.py b/test/web_socket/list-data.py index b71f328..9780f93 100644 --- a/test/web_socket/list-data.py +++ b/test/web_socket/list-data.py @@ -56,7 +56,7 @@ async def send_data_async(context: edge.SocketContext): } id += 1 print( - f'Send data to {context.message.session_id} in {datetime.datetime.now()}') + f'Send data to {context.message.session_id} in {datetime.datetime.now()}',type(context)) try: await context.send_object_async(data) except: # ConnectionError @@ -73,8 +73,8 @@ async def send_data_async(context: edge.SocketContext): ######## -@app.socket_action() -async def process_message_async(context: edge.SocketContext): +@app.endpoint_action() +async def process_message_async(context: edge.EndPointContext): print( f'message of type {context.message.type} come from {context.message.session_id} in {datetime.datetime.now()}') msg = await context.read_message_async() From 457350dc204cbff78049aea877270982cacb2d8b Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:41:55 +0330 Subject: [PATCH 09/15] fix bug about python 3.13 --- bclib/__init__.py | 28 +++++++ bclib/cache/__init__.py | 3 - bclib/cache/cache_item/function_cache_item.py | 7 +- bclib/cache/cache_item/scalar_cache_item.py | 5 +- bclib/cache/cache_manager.py | 25 +++++++ bclib/cache/factory.py | 14 ++-- bclib/cache/in_memory_cache_manager.py | 35 ++++----- bclib/cache/manager.py | 23 ------ bclib/cache/no_cache.py | 4 +- bclib/cache/signal_base_cache_manager.py | 33 +++++---- bclib/cache/signaler/factory.py | 16 ++-- bclib/cache/signaler/no_signaler.py | 5 +- bclib/cache/signaler/rabbit_signaler.py | 6 +- bclib/cache/value_item/array_value_item.py | 4 +- bclib/cache/value_item/base_value_item.py | 16 ++-- bclib/cache/value_item/scalar_value_item.py | 5 +- bclib/context/__init__.py | 15 ---- bclib/context/client_source_context.py | 6 +- bclib/context/client_source_member_context.py | 17 +++-- bclib/context/context.py | 17 +++-- bclib/context/json_base_request_context.py | 13 ++-- bclib/context/rabbit_context.py | 9 ++- bclib/context/request_context.py | 13 ++-- bclib/context/restful_context.py | 12 +-- bclib/context/server_source_context.py | 10 +-- bclib/context/server_source_member_context.py | 21 +++--- bclib/context/socket_context.py | 15 ++-- bclib/context/source_context.py | 2 +- bclib/context/source_member_context.py | 4 +- bclib/context/web_context.py | 49 ++++++------ bclib/db_manager/db_manager.py | 12 +-- bclib/db_manager/mongo_db.py | 3 - bclib/db_manager/odbc_db.py | 2 +- bclib/db_manager/rabbit_connection.py | 2 +- bclib/db_manager/restful_connection.py | 2 +- bclib/db_manager/sql_db.py | 2 +- bclib/db_manager/sqlite_db.py | 2 +- bclib/dispatcher/__init__.py | 7 -- bclib/dispatcher/callback_info.py | 13 ++-- bclib/dispatcher/dev_server_dispatcher.py | 14 ++-- bclib/dispatcher/dispatcher.py | 48 +++++++----- bclib/dispatcher/dispatcher_helper.py | 2 +- bclib/dispatcher/endpoint_dispatcher.py | 6 +- bclib/dispatcher/idispatcher.py | 12 +-- bclib/dispatcher/routing_dispatcher.py | 14 ++-- bclib/dispatcher/socket_dispatcher.py | 12 +-- bclib/edge.py | 74 ++++++++++++++----- bclib/edge_container.py | 61 ++++++++------- bclib/exception/__init__.py | 2 +- bclib/exception/bad_request_err.py | 2 +- bclib/exception/forbidden_err.py | 2 +- bclib/exception/handler_not_found_err.py | 2 +- bclib/exception/internal_server_err.py | 2 +- bclib/exception/not_found_err.py | 2 +- bclib/exception/unauthorized_err.py | 2 +- bclib/listener/__init__.py | 11 --- bclib/listener/http_listener/http_listener.py | 53 +++++++------ bclib/listener/rabbit_bus_listener.py | 4 +- bclib/listener/stream_base_message.py | 15 ++-- bclib/logger/__init__.py | 3 - bclib/logger/ilogger.py | 19 +++-- bclib/logger/logger_factory.py | 8 +- bclib/logger/no_logger.py | 8 +- bclib/logger/rabbit_schema_base_logger.py | 14 ++-- bclib/logger/restful_schema_base_logger.py | 2 +- bclib/logger/schema_base_logger.py | 4 +- bclib/parser/__init__.py | 2 +- bclib/parser/answer/answer.py | 50 +++++++------ bclib/parser/answer/question_data.py | 7 +- bclib/parser/answer/user_action.py | 26 +++---- bclib/parser/html/html_parser_ex.py | 2 +- bclib/predicate/all.py | 8 +- bclib/predicate/any.py | 8 +- bclib/predicate/between.py | 10 ++- bclib/predicate/callback.py | 12 +-- bclib/predicate/equal.py | 10 ++- bclib/predicate/greater_than.py | 13 ++-- bclib/predicate/greater_than_equal.py | 12 +-- bclib/predicate/has_value.py | 14 ++-- bclib/predicate/in_list.py | 14 ++-- bclib/predicate/less_than.py | 15 ++-- bclib/predicate/less_than_equal.py | 15 ++-- bclib/predicate/match.py | 15 ++-- bclib/predicate/not_equal.py | 14 ++-- bclib/predicate/predicate.py | 13 ++-- bclib/predicate/url.py | 15 ++-- 86 files changed, 640 insertions(+), 510 deletions(-) create mode 100644 bclib/cache/cache_manager.py delete mode 100644 bclib/cache/manager.py diff --git a/bclib/__init__.py b/bclib/__init__.py index edd5750..d04a09b 100644 --- a/bclib/__init__.py +++ b/bclib/__init__.py @@ -1 +1,29 @@ __version__ = "3.34.1" + +import bclib.context + +from bclib.listener.message import Message +from bclib.listener.message_type import MessageType +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + +from bclib.predicate import Predicate +from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders + +from bclib.db_manager import * + +# from bclib.context.client_source_context import ClientSourceContext +# from bclib.context.client_source_member_context import ClientSourceMemberContext +# from bclib.context.context import Context +# from bclib.context.restful_context import RESTfulContext +# from bclib.context.web_context import WebContext +# from bclib.context.request_context import RequestContext +# from bclib.context.rabbit_context import RabbitContext +# from bclib.context.socket_context import SocketContext +# from bclib.context.merge_type import MergeType +# from bclib.context.server_source_context import ServerSourceContext +# from bclib.context.server_source_member_context import ServerSourceMemberContext +# from bclib.context.source_context import SourceContext +# from bclib.context.source_member_context import SourceMemberContext +# from bclib.context.context_factory import ContextFactory +# from bclib.context.end_point_context import EndPointContext diff --git a/bclib/cache/__init__.py b/bclib/cache/__init__.py index 1cf4414..e69de29 100644 --- a/bclib/cache/__init__.py +++ b/bclib/cache/__init__.py @@ -1,3 +0,0 @@ -from bclib.cache.manager import CacheManager -from bclib.cache.factory import CacheFactory -from bclib.cache.cache_status import CacheStatus \ No newline at end of file diff --git a/bclib/cache/cache_item/function_cache_item.py b/bclib/cache/cache_item/function_cache_item.py index 40d9bd5..30f2c76 100644 --- a/bclib/cache/cache_item/function_cache_item.py +++ b/bclib/cache/cache_item/function_cache_item.py @@ -1,8 +1,9 @@ -from ..cache_item.base_cache_item import BaseCacheItem from typing import Callable +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class FunctionCacheItem(BaseCacheItem): - def __init__(self, data: "any", life_time:"int", function:"Callable") -> None: + def __init__(self, data: "any", life_time: "int", function: "Callable") -> None: super().__init__(data, life_time) self.__function = function @@ -11,4 +12,4 @@ def get_data(self, *args, **kwargs) -> "any": if data is None: data = self.__function(*args, **kwargs) self._update_data(data) - return data \ No newline at end of file + return data diff --git a/bclib/cache/cache_item/scalar_cache_item.py b/bclib/cache/cache_item/scalar_cache_item.py index 175bb47..1537d29 100644 --- a/bclib/cache/cache_item/scalar_cache_item.py +++ b/bclib/cache/cache_item/scalar_cache_item.py @@ -1,5 +1,6 @@ -from .base_cache_item import BaseCacheItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class ScalarCacheItem(BaseCacheItem): def __init__(self, data: "any", life_time: int) -> None: - super().__init__(data, life_time) \ No newline at end of file + super().__init__(data, life_time) diff --git a/bclib/cache/cache_manager.py b/bclib/cache/cache_manager.py new file mode 100644 index 0000000..330b862 --- /dev/null +++ b/bclib/cache/cache_manager.py @@ -0,0 +1,25 @@ +from abc import ABC, abstractmethod +from bclib.utility import DictEx +from bclib.cache.cache_status import CacheStatus + + +class CacheManager(ABC): + def __init__(self, options: "DictEx") -> None: + super().__init__() + self._options = options + + @abstractmethod + def cache_decorator(self, key: "str" = None, life_time: "int" = 0): ... + + @abstractmethod + def get_cache(self, key: "str") -> "list|any|None": ... + + @abstractmethod + def add_or_update(self, key: "str", data: "any", + life_time: "int" = 0) -> "CacheStatus": ... + + @abstractmethod + def clean(self) -> "CacheStatus": ... + + @abstractmethod + def reset(self, keys: "list[str]" = None) -> "CacheStatus": ... diff --git a/bclib/cache/factory.py b/bclib/cache/factory.py index 94fae8b..d267101 100644 --- a/bclib/cache/factory.py +++ b/bclib/cache/factory.py @@ -1,16 +1,18 @@ -from ..cache.manager import CacheManager +from cache.cache_manager import CacheManager from abc import ABC from bclib.utility import DictEx -from ..cache.in_memory_cache_manager import InMemoryCacheManager -from ..cache.no_cache import NoCacheManager +from bclib.cache.in_memory_cache_manager import InMemoryCacheManager +from bclib.cache.no_cache import NoCacheManager + class CacheFactory(ABC): @staticmethod - def create(options:"DictEx"=None) -> "CacheManager": - cache_type = str(options.type) if options is not None and options.has("type") else None + def create(options: "DictEx" = None) -> "CacheManager": + cache_type = str(options.type) if options is not None and options.has( + "type") else None if cache_type is not None: if cache_type == "memory": return InMemoryCacheManager(options) else: raise ValueError(f"Unknown type for cache ('${cache_type}')") - return NoCacheManager(options) \ No newline at end of file + return NoCacheManager(options) diff --git a/bclib/cache/in_memory_cache_manager.py b/bclib/cache/in_memory_cache_manager.py index 45e73a2..86e345f 100644 --- a/bclib/cache/in_memory_cache_manager.py +++ b/bclib/cache/in_memory_cache_manager.py @@ -1,23 +1,24 @@ from bclib.cache.cache_status import CacheStatus -from ..cache.signal_base_cache_manager import SignalBaseCacheManager +from bclib.cache.signal_base_cache_manager import SignalBaseCacheManager from bclib.utility import DictEx -from ..cache.cache_status import CacheStatus +from bclib.cache.cache_status import CacheStatus from typing import Callable -from ..cache.cache_item.base_cache_item import BaseCacheItem -from ..cache.cache_item.function_cache_item import FunctionCacheItem -from .cache_item.scalar_cache_item import ScalarCacheItem -from ..cache.value_item.base_value_item import BaseValueItem -from ..cache.value_item.array_value_item import ArrayValueItem -from ..cache.value_item.scalar_value_item import ScalarValueItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem +from bclib.cache.cache_item.function_cache_item import FunctionCacheItem +from bclib.cache.cache_item.scalar_cache_item import ScalarCacheItem +from bclib.cache.value_item.base_value_item import BaseValueItem +from bclib.cache.value_item.array_value_item import ArrayValueItem +from bclib.cache.value_item.scalar_value_item import ScalarValueItem from functools import wraps + class InMemoryCacheManager(SignalBaseCacheManager): - + def __init__(self, options: DictEx) -> None: super().__init__(options) - self.__cache_dict:"dict[str, BaseValueItem]" = dict() + self.__cache_dict: "dict[str, BaseValueItem]" = dict() - def __add_or_update(self, key:"str", cache_item:"BaseCacheItem", value_item:"BaseValueItem") -> "CacheStatus": + def __add_or_update(self, key: "str", cache_item: "BaseCacheItem", value_item: "BaseValueItem") -> "CacheStatus": if key not in self.__cache_dict: self.__cache_dict[key] = value_item(cache_item) return CacheStatus.ADDED @@ -29,7 +30,7 @@ def __add_or_update(self, key:"str", cache_item:"BaseCacheItem", value_item:"Bas print(repr(ex)) return CacheStatus.ERROR - def cache_decorator(self, key:"str"=None, life_time:"int"=0) -> "Callable": + def cache_decorator(self, key: "str" = None, life_time: "int" = 0) -> "Callable": """ Decorator that caches the result of a function for a specified key and life time. @@ -51,12 +52,12 @@ def decorator(function): @wraps(function) def wrapper(*args, **kwargs): - function_cache:"FunctionCacheItem" = function.cache + function_cache: "FunctionCacheItem" = function.cache return function_cache.get_data(*args, **kwargs) return wrapper return decorator - def reset(self, keys:"list[str]"=None) -> "CacheStatus": + def reset(self, keys: "list[str]" = None) -> "CacheStatus": """ Resets the cache by removing all items or specified keys. @@ -86,10 +87,10 @@ def clean(self) -> "CacheStatus": self.__cache_dict = cleaned_cache_dict return CacheStatus.CLEANED - def get_cache(self, key:"str") -> "list|any|None": + def get_cache(self, key: "str") -> "list|any|None": return self.__cache_dict[key].get_item() if key in self.__cache_dict else None - - def add_or_update(self, key: str, data: "any", life_time:"int"= 0) -> "CacheStatus": + + def add_or_update(self, key: str, data: "any", life_time: "int" = 0) -> "CacheStatus": """ Add or update an item in the cache. Args: diff --git a/bclib/cache/manager.py b/bclib/cache/manager.py deleted file mode 100644 index 322e0cf..0000000 --- a/bclib/cache/manager.py +++ /dev/null @@ -1,23 +0,0 @@ -from abc import ABC, abstractmethod -from bclib.utility import DictEx -from .cache_status import CacheStatus - -class CacheManager(ABC): - def __init__(self, options:"DictEx") -> None: - super().__init__() - self._options = options - - @abstractmethod - def cache_decorator(self, key:"str"=None, life_time:"int"=0): ... - - @abstractmethod - def get_cache(self, key:"str") -> "list|any|None": ... - - @abstractmethod - def add_or_update(self, key:"str", data:"any", life_time:"int"=0) -> "CacheStatus": ... - - @abstractmethod - def clean(self) -> "CacheStatus": ... - - @abstractmethod - def reset(self, keys:"list[str]"=None) -> "CacheStatus": ... diff --git a/bclib/cache/no_cache.py b/bclib/cache/no_cache.py index 8ce35e8..76251d4 100644 --- a/bclib/cache/no_cache.py +++ b/bclib/cache/no_cache.py @@ -1,5 +1,5 @@ -from ..cache.manager import CacheManager -from ..cache.cache_status import CacheStatus +from cache.cache_manager import CacheManager +from bclib.cache.cache_status import CacheStatus class NoCacheManager(CacheManager): diff --git a/bclib/cache/signal_base_cache_manager.py b/bclib/cache/signal_base_cache_manager.py index fb04b17..f20320c 100644 --- a/bclib/cache/signal_base_cache_manager.py +++ b/bclib/cache/signal_base_cache_manager.py @@ -1,39 +1,46 @@ -from ..cache.manager import CacheManager +from cache.cache_manager import CacheManager from bclib.utility import DictEx -from ..cache.signaler.factory import SignalerFactory +from bclib.cache.signaler.factory import SignalerFactory import asyncio + class SignalBaseCacheManager(CacheManager): - DEFAULT_CLEAN_INTERVAL = 43200 #Seconds => 12 Hours; 0 for indefinitely - DEFAULT_RESET_INTERVAL = 86400 #Seconds => 24 Hours; 0 for indefinitely + DEFAULT_CLEAN_INTERVAL = 43200 # Seconds => 12 Hours; 0 for indefinitely + DEFAULT_RESET_INTERVAL = 86400 # Seconds => 24 Hours; 0 for indefinitely def __init__(self, options: "DictEx") -> None: super().__init__(options) - self.__clean_interval = int(self._options.clean_interval) if self._options.has("clean_interval") else SignalBaseCacheManager.DEFAULT_CLEAN_INTERVAL + self.__clean_interval = int(self._options.clean_interval) if self._options.has( + "clean_interval") else SignalBaseCacheManager.DEFAULT_CLEAN_INTERVAL if self.__clean_interval < 0: raise ValueError("Invalid input for clean_interval!") - self.__reset_interval = int(self._options.reset_interval) if self._options.has("reset_interval") else SignalBaseCacheManager.DEFAULT_RESET_INTERVAL + self.__reset_interval = int(self._options.reset_interval) if self._options.has( + "reset_interval") else SignalBaseCacheManager.DEFAULT_RESET_INTERVAL if self.__reset_interval < 0: raise ValueError("Invalid input for reset_interval!") - signaler_options = self._options.signaler if self._options.has("signaler") else None - self._reset_signaler = SignalerFactory.create(self.reset, signaler_options) + signaler_options = self._options.signaler if self._options.has( + "signaler") else None + self._reset_signaler = SignalerFactory.create( + self.reset, signaler_options) if self.__reset_interval > 0 or self.__clean_interval > 0: loop = asyncio.get_event_loop() if self.__reset_interval > 0: loop.create_task(self.__reset_async(self.__reset_interval)) if self.__clean_interval > 0: loop.create_task(self.__clean_async(self.__clean_interval)) - - async def __reset_async(self, interval:"int"): + + async def __reset_async(self, interval: "int"): try: while True: await asyncio.sleep(interval) self.reset() - except: ... + except: + ... - async def __clean_async(self, interval:"int"): + async def __clean_async(self, interval: "int"): try: while True: await asyncio.sleep(interval) self.clean() - except: ... + except: + ... diff --git a/bclib/cache/signaler/factory.py b/bclib/cache/signaler/factory.py index abe5c31..0a43e1e 100644 --- a/bclib/cache/signaler/factory.py +++ b/bclib/cache/signaler/factory.py @@ -1,13 +1,14 @@ from abc import ABC -from ..signaler.base_signaler import BaseSignaler -from bclib.utility import DictEx -from ..signaler.no_signaler import NoSignaler -from ..signaler.rabbit_signaler import RabbitSignaller from typing import Callable +from bclib.cache.signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.no_signaler import NoSignaler +from bclib.cache.signaler.rabbit_signaler import RabbitSignaller +from bclib.utility import DictEx + class SignalerFactory(ABC): @staticmethod - def create(reset_cache_callback:"Callable", signaler_options:"DictEx"=None) -> "BaseSignaler": + def create(reset_cache_callback: "Callable", signaler_options: "DictEx" = None) -> "BaseSignaler": """ This is a factory function for create signaler @@ -26,5 +27,6 @@ def create(reset_cache_callback:"Callable", signaler_options:"DictEx"=None) -> " if signaler_type == "rabbit": return RabbitSignaller(reset_cache_callback, signaler_options) else: - raise ValueError(f"Unknown type for signaler ('${signaler_type}')") - return NoSignaler() \ No newline at end of file + raise ValueError( + f"Unknown type for signaler ('${signaler_type}')") + return NoSignaler() diff --git a/bclib/cache/signaler/no_signaler.py b/bclib/cache/signaler/no_signaler.py index 549e9cf..72a78d7 100644 --- a/bclib/cache/signaler/no_signaler.py +++ b/bclib/cache/signaler/no_signaler.py @@ -1,7 +1,8 @@ -from ..signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.base_signaler import BaseSignaler + class NoSignaler(BaseSignaler): """Implement no signaller for cache""" def __init__(self) -> None: - super().__init__(None, None) \ No newline at end of file + super().__init__(None, None) diff --git a/bclib/cache/signaler/rabbit_signaler.py b/bclib/cache/signaler/rabbit_signaler.py index cd23d25..12bf2b6 100644 --- a/bclib/cache/signaler/rabbit_signaler.py +++ b/bclib/cache/signaler/rabbit_signaler.py @@ -1,12 +1,14 @@ from typing import Callable import json import asyncio -from ..signaler.base_signaler import BaseSignaler +from bclib.cache.signaler.base_signaler import BaseSignaler from bclib.utility import DictEx + class RabbitSignaller(BaseSignaler): """Implement rabbit-mq signaler""" - def __init__(self, reset_cache_callback:"Callable", options:"DictEx") -> None: + + def __init__(self, reset_cache_callback: "Callable", options: "DictEx") -> None: super().__init__(reset_cache_callback, options) import pika try: diff --git a/bclib/cache/value_item/array_value_item.py b/bclib/cache/value_item/array_value_item.py index 77591c5..a0a5cee 100644 --- a/bclib/cache/value_item/array_value_item.py +++ b/bclib/cache/value_item/array_value_item.py @@ -1,6 +1,6 @@ from bclib.cache.cache_item.base_cache_item import BaseCacheItem -from ..value_item.base_value_item import BaseValueItem -from ..cache_item.function_cache_item import FunctionCacheItem +from bclib.cache.value_item.base_value_item import BaseValueItem + class ArrayValueItem(BaseValueItem): diff --git a/bclib/cache/value_item/base_value_item.py b/bclib/cache/value_item/base_value_item.py index f236026..e7d8368 100644 --- a/bclib/cache/value_item/base_value_item.py +++ b/bclib/cache/value_item/base_value_item.py @@ -1,18 +1,19 @@ from abc import ABC, abstractmethod -from ..cache_item.base_cache_item import BaseCacheItem +from bclib.cache.cache_item.base_cache_item import BaseCacheItem + class BaseValueItem(ABC): - def __init__(self, cache_item:"BaseCacheItem") -> None: + def __init__(self, cache_item: "BaseCacheItem") -> None: super().__init__() - self._item:"list[BaseCacheItem]|BaseCacheItem" = None + self._item: "list[BaseCacheItem]|BaseCacheItem" = None self._apply_item(cache_item) - + @abstractmethod - def _apply_item(self, cache_item:"BaseCacheItem"): ... + def _apply_item(self, cache_item: "BaseCacheItem"): ... def add_or_update_item(self, cache_item: "BaseCacheItem"): self._apply_item(cache_item) - + def get_item(self) -> "list|any|None": if self._item is not None: if isinstance(self._item, list): @@ -27,7 +28,6 @@ def get_item(self) -> "list|any|None": else: ret_val = None return ret_val - + def reset(self): self._item = None - diff --git a/bclib/cache/value_item/scalar_value_item.py b/bclib/cache/value_item/scalar_value_item.py index b3dd327..58c50f1 100644 --- a/bclib/cache/value_item/scalar_value_item.py +++ b/bclib/cache/value_item/scalar_value_item.py @@ -1,7 +1,8 @@ from bclib.cache.cache_item.base_cache_item import BaseCacheItem -from ..value_item.base_value_item import BaseValueItem +from bclib.cache.value_item.base_value_item import BaseValueItem + class ScalarValueItem(BaseValueItem): - + def _apply_item(self, cache_item: "BaseCacheItem") -> "BaseCacheItem": self._item = cache_item diff --git a/bclib/context/__init__.py b/bclib/context/__init__.py index 9d955be..e69de29 100644 --- a/bclib/context/__init__.py +++ b/bclib/context/__init__.py @@ -1,15 +0,0 @@ -from bclib.context.client_source_context import ClientSourceContext -from bclib.context.client_source_member_context import ClientSourceMemberContext -from bclib.context.context import Context -from bclib.context.restful_context import RESTfulContext -from bclib.context.web_context import WebContext -from bclib.context.request_context import RequestContext -from bclib.context.rabbit_context import RabbitContext -from bclib.context.socket_context import SocketContext -from bclib.context.merge_type import MergeType -from bclib.context.server_source_context import ServerSourceContext -from bclib.context.server_source_member_context import ServerSourceMemberContext -from bclib.context.source_context import SourceContext -from bclib.context.source_member_context import SourceMemberContext -from bclib.context.context_factory import ContextFactory -from bclib.context.end_point_context import EndPointContext diff --git a/bclib/context/client_source_context.py b/bclib/context/client_source_context.py index 738cd07..2f74200 100644 --- a/bclib/context/client_source_context.py +++ b/bclib/context/client_source_context.py @@ -2,7 +2,7 @@ from bclib.parser import HtmlParserEx from bclib.utility import DictEx -from .json_base_request_context import JsonBaseRequestContext +from bclib.context.json_base_request_context import JsonBaseRequestContext if TYPE_CHECKING: from bclib.dispatcher import IDispatcher @@ -12,8 +12,8 @@ class ClientSourceContext(JsonBaseRequestContext): """Context for client dbSource request""" - def __init__(self, cms_object: dict, dispatcher: 'IDispatcher',message_object: 'WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) parser = HtmlParserEx() self.raw_command = self.cms.form.command self.dmn_id = self.cms.form.dmnid if self.cms.form.has( diff --git a/bclib/context/client_source_member_context.py b/bclib/context/client_source_member_context.py index 5faea5a..40d94dc 100644 --- a/bclib/context/client_source_member_context.py +++ b/bclib/context/client_source_member_context.py @@ -1,22 +1,23 @@ -from abc import ABC from typing import Any -from .merge_type import MergeType -from .context import Context -from .client_source_context import ClientSourceContext +from bclib.context.merge_type import MergeType +from bclib.context.context import Context +from bclib.context.client_source_context import ClientSourceContext + +print(__name__) class ClientSourceMemberContext(Context): """Context for dbSource member request""" - def __init__(self, sourceContext: ClientSourceContext, data: Any, member: dict) -> None: - super().__init__(sourceContext.dispatcher) - self.__source_context = sourceContext + def __init__(self, source_context: 'ClientSourceContext', data: 'Any', member: 'dict') -> None: + super().__init__(source_context.dispatcher) + self.__source_context = source_context self.member = member self.data = data self.cms = self.__source_context.cms self.command = self.__source_context.command self.url = self.__source_context.url - self.table_name: str = f"{sourceContext.command.name}.{member.name}" + self.table_name: str = f"{source_context.command.name}.{member.name}" self.key_field_name: str = None self.status_field_name: str = None self.merge_type: MergeType = MergeType.REPLACE diff --git a/bclib/context/context.py b/bclib/context/context.py index 9f42cda..186a0cb 100644 --- a/bclib/context/context.py +++ b/bclib/context/context.py @@ -14,14 +14,14 @@ class Context(ABC): """Base class for dispatching""" - def __init__(self, dispatcher: 'IDispatcher') -> None: + def __init__(self, dispatcher: 'IDispatcher') -> 'None': super().__init__() self.dispatcher = dispatcher - self.url_segments: DictEx = None - self.url: Optional[str] = None + self.url_segments: 'DictEx' = None + self.url: 'Optional[str]' = None self.is_adhoc = True - #TODO:Removed wrapper open_X_connection methode + # TODO:Removed wrapper open_X_connection methode def generate_error_response(self, exception: Exception) -> dict: """Generate error response from process result""" @@ -45,8 +45,8 @@ def _generate_error_object(self, exception: Exception) -> 'Tuple[dict, str]': "errorCode": error_code, "errorMessage": str(exception) } - #TODO:use logger service - if True:#self.dispatcher.log_error: + # TODO:use logger service + if True: # self.dispatcher.log_error: error_object["error"] = traceback.format_exc() return (error_object, status_code) @@ -68,8 +68,9 @@ def _generate_response_cms( ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.INDEX] = response_type ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.HEADER_CODE] = status_code ret_val[HttpBaseDataType.CMS][HttpBaseDataName.WEB_SERVER][HttpBaseDataName.MIME] = mime - if isinstance(content,bytes): - ret_val[HttpBaseDataType.CMS][HttpBaseDataName.BLOB_CONTENT] = base64.b64encode(content).decode("utf-8") + if isinstance(content, bytes): + ret_val[HttpBaseDataType.CMS][HttpBaseDataName.BLOB_CONTENT] = base64.b64encode( + content).decode("utf-8") else: ret_val[HttpBaseDataType.CMS][HttpBaseDataName.CONTENT] = content if headers is not None: diff --git a/bclib/context/json_base_request_context.py b/bclib/context/json_base_request_context.py index bbedd33..0407cb3 100644 --- a/bclib/context/json_base_request_context.py +++ b/bclib/context/json_base_request_context.py @@ -2,26 +2,27 @@ from typing import TYPE_CHECKING from bclib.utility import HttpMimeTypes -from .web_context import WebContext +from bclib.context.web_context import WebContext if TYPE_CHECKING: - from bclib import dispatcher,listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.web_message import WebMessage class JsonBaseRequestContext(WebContext): """Base class for dispatching http json base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) self.mime = HttpMimeTypes.JSON - def generate_error_response(self, exception: Exception) -> dict: + def generate_error_response(self, exception: 'Exception') -> 'dict': """Generate error response from process result""" error_object, self.status_code = self._generate_error_object(exception) self.mime = HttpMimeTypes.JSON return self.generate_response(error_object) - def generate_response(self, content: dict) -> dict: + def generate_response(self, content: 'dict') -> 'dict': """Generate response from process content""" return super().generate_response(json.dumps(content)) diff --git a/bclib/context/rabbit_context.py b/bclib/context/rabbit_context.py index 411f13d..fd4993d 100644 --- a/bclib/context/rabbit_context.py +++ b/bclib/context/rabbit_context.py @@ -1,9 +1,10 @@ import json -from typing import Any, TYPE_CHECKING -from .context import Context +from typing import TYPE_CHECKING +from bclib.context.context import Context +from bclib.utility.dict_ex import DictEx + if TYPE_CHECKING: - from bclib.dispatcher import IDispatcher -from bclib.utility import DictEx + from bclib.dispatcher.idispatcher import IDispatcher class RabbitContext(Context): diff --git a/bclib/context/request_context.py b/bclib/context/request_context.py index e4fd3a4..cbb4cf6 100644 --- a/bclib/context/request_context.py +++ b/bclib/context/request_context.py @@ -1,24 +1,24 @@ import json -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Optional from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes from bclib.exception import ShortCircuitErr -from .context import Context +from bclib.context.context import Context if TYPE_CHECKING: - from bclib import dispatcher + from bclib.dispatcher.idispatcher import IDispatcher class RequestContext(Context): """Base class for dispatching http base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher') -> None: super().__init__(dispatcher) self.cms = DictEx(cms_object) self.url: str = self.cms.request.url self.query: DictEx = self.cms.query self.form: DictEx = self.cms.form - self.__headers: dict = None + self.__headers: Optional[dict] = None self.response_type: str = ResponseTypes.RENDERED self.status_code: str = HttpStatusCodes.OK self.mime = HttpMimeTypes.HTML @@ -40,7 +40,8 @@ def generate_error_response(self, exception: Exception) -> dict: content = exception.data if isinstance( exception.data, str) else json.dumps(exception.data, indent=1).replace("\n", "
") else: - content = f"{error_object['errorMessage']} (Error Code: {error_object['errorCode']})" + content = f"{error_object['errorMessage']} (Error Code: " +\ + f"{error_object['errorCode']})" if 'error' in error_object: error = error_object["error"].replace("\n", "
") content += f"
{error}" diff --git a/bclib/context/restful_context.py b/bclib/context/restful_context.py index eb3cdf8..68ce0ad 100644 --- a/bclib/context/restful_context.py +++ b/bclib/context/restful_context.py @@ -1,17 +1,17 @@ import json from typing import TYPE_CHECKING -from bclib.utility import DictEx -from ..context.json_base_request_context import JsonBaseRequestContext +from bclib.utility.dict_ex import DictEx +from bclib.context.json_base_request_context import JsonBaseRequestContext from urllib.parse import parse_qsl if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.web_message import WebMessage class RESTfulContext(JsonBaseRequestContext): - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher',message_object: 'listener.WebMessage') -> None: - super().__init__(cms_object, dispatcher,message_object) + def __init__(self, cms_object: 'dict', dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: + super().__init__(cms_object, dispatcher, message_object) temp_data = None if self.cms.form: temp_data = self.cms.form diff --git a/bclib/context/server_source_context.py b/bclib/context/server_source_context.py index 4cb49f2..17b4b02 100644 --- a/bclib/context/server_source_context.py +++ b/bclib/context/server_source_context.py @@ -1,17 +1,17 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from .. import dispatcher + from bclib.dispatcher.idispatcher import IDispatcher -from bclib.parser import HtmlParserEx -from bclib.utility import DictEx -from ..context.context import Context +from bclib.parser.html.html_parser_ex import HtmlParserEx +from bclib.utility.dict_ex import DictEx +from bclib.context.context import Context class ServerSourceContext(Context): """Base class for dispatching server base dbsource request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher') -> None: super().__init__(dispatcher) parser = HtmlParserEx() self.raw_command = cms_object["command"] diff --git a/bclib/context/server_source_member_context.py b/bclib/context/server_source_member_context.py index 0ecc264..f864ec4 100644 --- a/bclib/context/server_source_member_context.py +++ b/bclib/context/server_source_member_context.py @@ -1,20 +1,21 @@ -from typing import Any -from ..context.merge_type import MergeType -from ..context.context import Context -from ..context.server_source_context import ServerSourceContext +from typing import Any, TYPE_CHECKING, Optional +from bclib.context.merge_type import MergeType +from bclib.context.context import Context +if TYPE_CHECKING: + from bclib.context.server_source_context import ServerSourceContext class ServerSourceMemberContext(Context): """Context for Server dbSource member request""" - def __init__(self, sourceContext: ServerSourceContext, data: Any, member: dict) -> None: - super().__init__(sourceContext.dispatcher) - self.__source_context = sourceContext + def __init__(self, source_context: 'ServerSourceContext', data: Any, member: 'dict') -> None: + super().__init__(source_context.dispatcher) + self.__source_context = source_context self.member = member self.data = data self.command = self.__source_context.command - self.table_name: str = f"{sourceContext.command.name}.{member.name}" - self.key_field_name: str = None - self.status_field_name: str = None + self.table_name: str = f"{source_context.command.name}.{member.name}" + self.key_field_name: Optional[str] = None + self.status_field_name: Optional[str] = None self.merge_type: MergeType = MergeType.REPLACE self.column_names: 'list[str]' = None diff --git a/bclib/context/socket_context.py b/bclib/context/socket_context.py index ebbb9b6..bbc5395 100644 --- a/bclib/context/socket_context.py +++ b/bclib/context/socket_context.py @@ -2,18 +2,18 @@ from typing import TYPE_CHECKING from bclib.listener.message_type import MessageType -from bclib.context import Context +from bclib.context.context import Context from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, HttpStatusCodes, ResponseTypes if TYPE_CHECKING: - from .. import dispatcher - from .. import listener + from bclib.dispatcher.idispatcher import IDispatcher + from bclib.listener.socket_message import SocketMessage class SocketContext(Context): """Base class for dispatching web socket base request context""" - def __init__(self, cms_object: dict, dispatcher: 'dispatcher.IDispatcher', message_object: 'listener.SocketMessage', body: dict) -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'SocketMessage', body: dict) -> None: super().__init__(dispatcher) self.cms = DictEx(cms_object) if cms_object else None self.url: str = self.cms.request.url if cms_object else None @@ -39,11 +39,10 @@ async def send_content_async(self, headers: 'dict' = None) -> None: cms = Context._generate_response_cms( content, response_type, status_code, mime, template, headers) - await self.message.write_result_async(cms,MessageType.MESSAGE) + await self.message.write_result_async(cms, MessageType.MESSAGE) - async def read_message_async(self) -> 'listener.SocketMessage': + async def read_message_async(self) -> 'SocketMessage': return await self.message.read_next_message_async() async def send_close_async(self) -> None: - await self.message.write_result_async(None,MessageType.DISCONNECT) - + await self.message.write_result_async(None, MessageType.DISCONNECT) diff --git a/bclib/context/source_context.py b/bclib/context/source_context.py index cde6933..4adb15d 100644 --- a/bclib/context/source_context.py +++ b/bclib/context/source_context.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx class SourceContext(ABC): diff --git a/bclib/context/source_member_context.py b/bclib/context/source_member_context.py index 89503bb..c300358 100644 --- a/bclib/context/source_member_context.py +++ b/bclib/context/source_member_context.py @@ -1,9 +1,9 @@ from abc import ABC, abstractmethod from typing import Any -from ..context.merge_type import MergeType +from bclib.context.merge_type import MergeType -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx class SourceMemberContext(ABC): diff --git a/bclib/context/web_context.py b/bclib/context/web_context.py index f4c3c61..01b1eb6 100644 --- a/bclib/context/web_context.py +++ b/bclib/context/web_context.py @@ -3,50 +3,51 @@ from typing import Any, TYPE_CHECKING, Coroutine, Iterator, Optional, Union from aiohttp.web_response import ContentCoding -from .request_context import RequestContext +from bclib.context.request_context import RequestContext if TYPE_CHECKING: from bclib.listener import WebMessage + from bclib.dispatcher.idispatcher import IDispatcher class WebContext(RequestContext): - def __init__(self, cms_object: dict, dispatcher: 'IDispatcher',message_object: 'WebMessage') -> None: + def __init__(self, cms_object: dict, dispatcher: 'IDispatcher', message_object: 'WebMessage') -> None: super().__init__(cms_object, dispatcher) self.process_async = True self.__message = message_object - - async def start_stream_response_async(self,status: int = 200, - reason: Optional[str] = 'OK', - headers: Optional[dict] = None,)-> Coroutine[Any, Any, None]: - await self.__message.start_stream_response_async(status,reason,headers) - async def write_async(self,data:'bytes') -> Coroutine[Any, Any, None]: - await self.__message.write_async(data) + async def start_stream_response_async(self, status: int = 200, + reason: Optional[str] = 'OK', + headers: Optional[dict] = None,) -> Coroutine[Any, Any, None]: + await self.__message.start_stream_response_async(status, reason, headers) + + async def write_async(self, data: 'bytes') -> Coroutine[Any, Any, None]: + await self.__message.write_async(data) async def drain_async(self) -> Coroutine[Any, Any, None]: await self.__message.drain_async() - - async def write_and_drain_async(self,data:'bytes') -> Coroutine[Any, Any, None]: + + async def write_and_drain_async(self, data: 'bytes') -> Coroutine[Any, Any, None]: await self.write_async(data) await self.drain_async() - async def enable_compression(self,force: Optional[Union[bool, ContentCoding]] = None) -> None: + async def enable_compression(self, force: Optional[Union[bool, ContentCoding]] = None) -> None: await self.__message.enable_compression(force) - async def drain_array_async(self,data_list:Iterator,source_name:str, chunk_size:int,delimiter:str=',')-> Coroutine[Any, Any, None]: - total_len =len(data_list) - current:int = 0 + async def drain_array_async(self, data_list: Iterator, source_name: str, chunk_size: int, delimiter: str = ',') -> Coroutine[Any, Any, None]: + total_len = len(data_list) + current: int = 0 while current < total_len: - temp_list = list(islice(data_list,current,current + chunk_size)) - current+= len(temp_list) + temp_list = list(islice(data_list, current, current + chunk_size)) + current += len(temp_list) data = { "sources": [ - { - "options": { - "tableName": source_name, - "mergeType": 1 #MergeType append, - }, - "data": temp_list - }], + { + "options": { + "tableName": source_name, + "mergeType": 1 # MergeType append, + }, + "data": temp_list + }], } await self.write_and_drain_async(f"{json.dumps(data)}{delimiter}".encode()) diff --git a/bclib/db_manager/db_manager.py b/bclib/db_manager/db_manager.py index 75abfa0..a670c9f 100644 --- a/bclib/db_manager/db_manager.py +++ b/bclib/db_manager/db_manager.py @@ -1,11 +1,11 @@ import asyncio from bclib.utility import DictEx -from ..db_manager.rabbit_connection import RabbitConnection -from ..db_manager.db import Db -from ..db_manager.mongo_db import MongoDb -from ..db_manager.sql_db import SqlDb -from ..db_manager.sqlite_db import SQLiteDb -from ..db_manager.restful_connection import RESTfulConnection +from bclib.db_manager.rabbit_connection import RabbitConnection +from bclib.db_manager.db import Db +from bclib.db_manager.mongo_db import MongoDb +from bclib.db_manager.sql_db import SqlDb +from bclib.db_manager.sqlite_db import SQLiteDb +from bclib.db_manager.restful_connection import RESTfulConnection class DbManager: diff --git a/bclib/db_manager/mongo_db.py b/bclib/db_manager/mongo_db.py index eeebc08..590d237 100644 --- a/bclib/db_manager/mongo_db.py +++ b/bclib/db_manager/mongo_db.py @@ -1,6 +1,3 @@ -from ..db_manager.db import Db - - class SingletonMeta(type): """ The Singleton class can be implemented in different ways in Python. Some diff --git a/bclib/db_manager/odbc_db.py b/bclib/db_manager/odbc_db.py index f0ecbe5..afcaf40 100644 --- a/bclib/db_manager/odbc_db.py +++ b/bclib/db_manager/odbc_db.py @@ -2,7 +2,7 @@ Implementation of ODBC base Db object https://github.com/mkleehammer/pyodbc/wiki """ -from ..db_manager.db import Db +from bclib.db_manager.db import Db class OdbcDb(Db): diff --git a/bclib/db_manager/rabbit_connection.py b/bclib/db_manager/rabbit_connection.py index f1e8b99..a9bc8e1 100644 --- a/bclib/db_manager/rabbit_connection.py +++ b/bclib/db_manager/rabbit_connection.py @@ -2,7 +2,7 @@ import json from typing import Any from bclib.utility import DictEx -from ..db_manager.db import Db +from bclib.db_manager.db import Db class RabbitConnection(Db): diff --git a/bclib/db_manager/restful_connection.py b/bclib/db_manager/restful_connection.py index 48d4bc4..a818780 100644 --- a/bclib/db_manager/restful_connection.py +++ b/bclib/db_manager/restful_connection.py @@ -1,6 +1,6 @@ """RESTful implementation of Db wrapper""" from typing import Any -from ..db_manager.db import Db +from bclib.db_manager.db import Db class RESTfulConnection(Db): diff --git a/bclib/db_manager/sql_db.py b/bclib/db_manager/sql_db.py index 8877fa7..6399ccb 100644 --- a/bclib/db_manager/sql_db.py +++ b/bclib/db_manager/sql_db.py @@ -1,4 +1,4 @@ -from ..db_manager.odbc_db import OdbcDb +from bclib.db_manager.odbc_db import OdbcDb class SqlDb(OdbcDb): diff --git a/bclib/db_manager/sqlite_db.py b/bclib/db_manager/sqlite_db.py index 6fc48f3..df9ffa6 100644 --- a/bclib/db_manager/sqlite_db.py +++ b/bclib/db_manager/sqlite_db.py @@ -1,5 +1,5 @@ import sqlite3 -from ..db_manager.db import Db +from bclib.db_manager.db import Db class SQLiteDb(Db): diff --git a/bclib/dispatcher/__init__.py b/bclib/dispatcher/__init__.py index 2674718..e69de29 100644 --- a/bclib/dispatcher/__init__.py +++ b/bclib/dispatcher/__init__.py @@ -1,7 +0,0 @@ -from bclib.dispatcher.dispatcher import Dispatcher -from bclib.dispatcher.idispatcher import IDispatcher -from bclib.dispatcher.socket_dispatcher import SocketDispatcher -from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher -from bclib.dispatcher.routing_dispatcher import RoutingDispatcher -from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher -from bclib.dispatcher.dispatcher_helper import DispatcherHelper diff --git a/bclib/dispatcher/callback_info.py b/bclib/dispatcher/callback_info.py index 1ea3df0..0804d2b 100644 --- a/bclib/dispatcher/callback_info.py +++ b/bclib/dispatcher/callback_info.py @@ -1,15 +1,18 @@ -from typing import Any, Callable, Awaitable -from ..context import Context -from ..predicate import Predicate +from typing import TYPE_CHECKING from bclib.exception import ShortCircuitErr +if TYPE_CHECKING: + from typing import Any, Callable, Awaitable + from bclib.predicate import Predicate + from bclib.context.context import Context + class CallbackInfo: - def __init__(self, predicates: 'list[Predicate]', async_callback: 'Callable[[Context], Awaitable[Any]]') -> Any: + def __init__(self, predicates: 'list[Predicate]', async_callback: 'Callable[[Context], Awaitable[Any]]') -> 'Any': self.__async_callback = async_callback self.__predicates = predicates - async def try_execute_async(self, context: Context) -> Any: + async def try_execute_async(self, context: 'Context') -> 'Any': result: Any = None for predicate in self.__predicates: try: diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index 96e7680..3b01719 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -3,23 +3,25 @@ from context.context_factory import ContextFactory from dependency_injector import containers -from bclib.logger import ILogger -from bclib.cache import CacheManager +from bclib.logger.ilogger import ILogger +from cache.cache_manager import CacheManager from bclib.db_manager import DbManager from bclib.utility import DictEx -from bclib.listener import Endpoint, HttpListener +from bclib.listener.endpoint import Endpoint +from bclib.listener.http_listener import HttpListener from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class DevServerDispatcher(RoutingDispatcher): - def __init__(self,container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container=container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger = logger, loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__listener = HttpListener( Endpoint(self.options.server), self._on_message_receive_async, self.options.ssl, self.options.configuration - ) + ) def initialize_task(self): super().initialize_task() diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index 5303ea8..8d1c365 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -3,29 +3,39 @@ import inspect from abc import ABC import signal -import traceback -from typing import Awaitable, Callable, Any, Coroutine, Optional +from typing import Awaitable, Callable, Any, Coroutine, Optional, TYPE_CHECKING from functools import wraps from dependency_injector import containers -from bclib.logger import ILogger, LogObject -from bclib.cache import CacheManager -from bclib.listener import RabbitBusListener +from bclib.logger.ilogger import ILogger +from cache.cache_manager import CacheManager from bclib.predicate import Predicate -from bclib.db_manager import DbManager +from bclib.db_manager.db_manager import DbManager from bclib.utility import DictEx from bclib.exception import HandlerNotFoundErr -from .callback_info import CallbackInfo +from bclib.dispatcher.callback_info import CallbackInfo +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.client_source_member_context import ClientSourceMemberContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.rabbit_context import RabbitContext +from bclib.context.socket_context import SocketContext +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.server_source_member_context import ServerSourceMemberContext +from bclib.context.end_point_context import EndPointContext + +if TYPE_CHECKING: + from bclib.logger.ilogger import LogObject -from bclib.context import ClientSourceContext, ClientSourceMemberContext, WebContext, Context, RESTfulContext, RabbitContext, SocketContext, ServerSourceContext, ServerSourceMemberContext,EndPointContext class Dispatcher(ABC): """Base class for dispatching request""" - def __init__(self,container:'containers.Container', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger', loop:'asyncio.AbstractEventLoop'): + def __init__(self, container: 'containers.Container', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop'): self.options = options - self.container= container + self.container = container self.__look_up: 'dict[str, list[CallbackInfo]]' = dict() self.event_loop: 'asyncio.AbstractEventLoop' = loop self.cache_manager: 'CacheManager' = cache_manager @@ -59,7 +69,7 @@ async def async_wrapper(context: 'EndPointContext'): .append(CallbackInfo([*predicates], wrapper)) return end_point_action_handler return _decorator - + def socket_action(self, * predicates: (Predicate)): """Decorator for determine Socket action""" @@ -379,14 +389,14 @@ def initialize_task(self): for dispatcher in self.__rabbit_dispatcher: dispatcher.initialize_task(self.event_loop) - #TODO:pre ansd post callback replaced with resource provider of DI - def listening(self, with_block:bool = True): + # TODO:pre ansd post callback replaced with resource provider of DI + def listening(self, with_block: bool = True): """Start listening to request for process""" for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda sig, _: self.event_loop.stop()) init_process = self.container.init_resources() - if isinstance( init_process,Awaitable): - self.event_loop.run_until_complete( init_process) + if isinstance(init_process, Awaitable): + self.event_loop.run_until_complete(init_process) self.initialize_task() if with_block: self.event_loop.run_forever() @@ -396,14 +406,14 @@ def listening(self, with_block:bool = True): group = asyncio.gather(*tasks, return_exceptions=True) self.event_loop.run_until_complete(group) shutdown_process = self.container.shutdown_resources() - if isinstance( shutdown_process,Awaitable): + if isinstance(shutdown_process, Awaitable): self.event_loop.run_until_complete(shutdown_process) self.event_loop.close() - def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: + def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> 'LogObject': return self.logger.new_object_log(schema_name, routing_key, **kwargs) - async def log_async(self, log_object: LogObject = None, **kwargs): + async def log_async(self, log_object: 'LogObject' = None, **kwargs): """log params""" if log_object is None: if "schema_name" not in kwargs: @@ -412,7 +422,7 @@ async def log_async(self, log_object: LogObject = None, **kwargs): log_object = self.new_object_log(schema_name, **kwargs) await self.logger.log_async(log_object) - def log_in_background(self, log_object: LogObject = None, **kwargs) -> Coroutine: + def log_in_background(self, log_object: 'LogObject' = None, **kwargs) -> Coroutine: """log params in background precess""" return self.event_loop.create_task( self.log_async(log_object, **kwargs) diff --git a/bclib/dispatcher/dispatcher_helper.py b/bclib/dispatcher/dispatcher_helper.py index b72565d..a980c5b 100644 --- a/bclib/dispatcher/dispatcher_helper.py +++ b/bclib/dispatcher/dispatcher_helper.py @@ -3,7 +3,7 @@ from bclib.predicate import Predicate, InList, Equal, Url, Between, NotEqual, GreaterThan, LessThan, LessThanEqual, GreaterThanEqual, Match, HasValue, Callback, All from bclib import predicate -from bclib.context import Context +from bclib.context.context import Context class DispatcherHelper: diff --git a/bclib/dispatcher/endpoint_dispatcher.py b/bclib/dispatcher/endpoint_dispatcher.py index b9f19fa..2223184 100644 --- a/bclib/dispatcher/endpoint_dispatcher.py +++ b/bclib/dispatcher/endpoint_dispatcher.py @@ -3,11 +3,11 @@ from dependency_injector import containers from bclib.listener.end_point_message import EndPointMessage from bclib.context.context_factory import ContextFactory -from bclib.cache import CacheManager +from cache.cache_manager import CacheManager from bclib.db_manager import DbManager -from bclib.logger import ILogger +from bclib.logger.ilogger import ILogger from bclib.utility import DictEx -from bclib.listener import Endpoint +from bclib.listener.endpoint import Endpoint from bclib.dispatcher.routing_dispatcher import RoutingDispatcher diff --git a/bclib/dispatcher/idispatcher.py b/bclib/dispatcher/idispatcher.py index 5c52a7e..10be69f 100644 --- a/bclib/dispatcher/idispatcher.py +++ b/bclib/dispatcher/idispatcher.py @@ -4,18 +4,18 @@ from dependency_injector import containers from typing import Callable, Any, TYPE_CHECKING, Coroutine, Optional -from bclib.db_manager import DbManager -from bclib.cache import CacheManager +from bclib.db_manager.db_manager import DbManager +from cache.cache_manager import CacheManager from bclib.utility import DictEx -from bclib.logger import LogObject +from bclib.logger.log_object import LogObject if TYPE_CHECKING: - from context import Context + from bclib.context.context import Context class IDispatcher(ABC): """Dispatcher base class with core functionality for manage cache and background process""" - - container:'containers.Container' + + container: 'containers.Container' @property @abstractmethod diff --git a/bclib/dispatcher/routing_dispatcher.py b/bclib/dispatcher/routing_dispatcher.py index 29ab5f6..8eadadb 100644 --- a/bclib/dispatcher/routing_dispatcher.py +++ b/bclib/dispatcher/routing_dispatcher.py @@ -1,17 +1,17 @@ import asyncio import inspect -from typing import Callable, Any, Coroutine, Optional +from typing import Callable, Any, Coroutine from dependency_injector import containers -from bclib.logger import ILogger -from bclib.db_manager import DbManager -from bclib.cache import CacheManager +from bclib.logger.ilogger import ILogger +from bclib.db_manager.db_manager import DbManager +from cache.cache_manager import CacheManager from bclib.utility import DictEx from bclib.context.context_factory import ContextFactory -from bclib.listener import Message, MessageType -from .dispatcher_helper import DispatcherHelper -from .dispatcher import Dispatcher +from bclib.listener.message import Message, MessageType +from bclib.dispatcher.dispatcher_helper import DispatcherHelper +from bclib.dispatcher.dispatcher import Dispatcher class RoutingDispatcher(Dispatcher, DispatcherHelper): diff --git a/bclib/dispatcher/socket_dispatcher.py b/bclib/dispatcher/socket_dispatcher.py index fad18d4..d38987b 100644 --- a/bclib/dispatcher/socket_dispatcher.py +++ b/bclib/dispatcher/socket_dispatcher.py @@ -2,17 +2,19 @@ from dependency_injector import containers from bclib.context.context_factory import ContextFactory -from bclib.cache import CacheManager +from cache.cache_manager import CacheManager from bclib.db_manager import DbManager -from bclib.logger import ILogger +from bclib.logger.ilogger import ILogger from bclib.utility import DictEx -from bclib.listener import Endpoint, SocketListener +from bclib.listener.endpoint import Endpoint +from bclib.listener.socket_listener import SocketListener from bclib.dispatcher.routing_dispatcher import RoutingDispatcher class SocketDispatcher(RoutingDispatcher): - def __init__(self, container:'containers.Container', context_factory:'ContextFactory', options: 'DictEx',cache_manager:'CacheManager',db_manager:'DbManager',logger:'ILogger',loop:'asyncio.AbstractEventLoop'=None): - super().__init__(container= container,context_factory=context_factory, options=options,cache_manager=cache_manager,db_manager=db_manager,logger=logger,loop=loop) + def __init__(self, container: 'containers.Container', context_factory: 'ContextFactory', options: 'DictEx', cache_manager: 'CacheManager', db_manager: 'DbManager', logger: 'ILogger', loop: 'asyncio.AbstractEventLoop' = None): + super().__init__(container=container, context_factory=context_factory, options=options, + cache_manager=cache_manager, db_manager=db_manager, logger=logger, loop=loop) self.__listener = SocketListener( Endpoint(self.options.receiver), Endpoint(self.options.sender), diff --git a/bclib/edge.py b/bclib/edge.py index 77fe842..6546d82 100644 --- a/bclib/edge.py +++ b/bclib/edge.py @@ -2,16 +2,48 @@ import asyncio from dependency_injector import providers -from bclib.db_manager import * -from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher -from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext,EndPointContext +# from bclib.db_manager import * +# from bclib.dispatcher import RoutingDispatcher, IDispatcher, SocketDispatcher, DevServerDispatcher, EndpointDispatcher +# from bclib.context import Context, WebContext, SocketContext, ClientSourceContext, ClientSourceMemberContext, RabbitContext, RESTfulContext, RequestContext, MergeType, ServerSourceContext, ServerSourceMemberContext, SourceContext, SourceMemberContext, EndPointContext from bclib.utility import DictEx, HttpStatusCodes, HttpMimeTypes, ResponseTypes, HttpHeaders -from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName +# from bclib.listener import Message, MessageType, HttpBaseDataType, HttpBaseDataName from bclib.predicate import Predicate from bclib.exception import * from bclib.edge_container import EdgeContainer + +from bclib.db_manager import DbManager, SqlDb, SQLiteDb, MongoDb, RabbitConnection, RESTfulConnection + +from bclib.listener.message import Message +from bclib.listener.message_type import MessageType +from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName +from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType + + +from bclib.dispatcher.idispatcher import IDispatcher +from bclib.dispatcher.socket_dispatcher import SocketDispatcher +from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher +from bclib.dispatcher.routing_dispatcher import RoutingDispatcher +from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher + +from bclib.context.client_source_context import ClientSourceContext +from bclib.context.client_source_member_context import ClientSourceMemberContext +from bclib.context.context import Context +from bclib.context.restful_context import RESTfulContext +from bclib.context.web_context import WebContext +from bclib.context.request_context import RequestContext +from bclib.context.rabbit_context import RabbitContext +from bclib.context.socket_context import SocketContext +from bclib.context.merge_type import MergeType +from bclib.context.server_source_context import ServerSourceContext +from bclib.context.server_source_member_context import ServerSourceMemberContext +from bclib.context.source_context import SourceContext +from bclib.context.source_member_context import SourceMemberContext +from bclib.context.end_point_context import EndPointContext + + from bclib import __version__ + def from_config(option_file_path: str, file_name: str = "host.json"): """Create related RoutingDispatcher obj from config file in related path""" import json @@ -31,25 +63,27 @@ def from_list(hosts: 'dict[str,list[str]]'): __print_splash(True) loop = asyncio.get_event_loop() with concurrent.futures.ThreadPoolExecutor(max_workers=len(hosts.items())) as executor: - tasks:list[asyncio.Future] = [] + tasks: list[asyncio.Future] = [] for host, args in hosts.items(): - args.append(f"-n {host}") - args.append("-m") - tasks.append(loop.run_in_executor(executor, subprocess.run, args)) - print(f'{host} start running from {args[1]}') + args.append(f"-n {host}") + args.append("-m") + tasks.append(loop.run_in_executor(executor, subprocess.run, args)) + print(f'{host} start running from {args[1]}') try: loop.run_until_complete(asyncio.gather(*tasks)) except KeyboardInterrupt: pass -def from_options(options: dict,loop:asyncio.AbstractEventLoop = None) -> RoutingDispatcher: - container=EdgeContainer() + +def from_options(options: 'dict', loop: 'asyncio.AbstractEventLoop' = None) -> 'RoutingDispatcher': + container = EdgeContainer() container.app_config.from_dict(options) if loop: container.app_config.loop.from_value(loop) return from_container(container) -def __get_arg_parts(container:'EdgeContainer'): + +def __get_arg_parts(container: 'EdgeContainer'): import sys import getopt @@ -71,7 +105,8 @@ def __get_arg_parts(container:'EdgeContainer'): except getopt.error as err: print(str(err)) -def from_container(container:'EdgeContainer') -> RoutingDispatcher: + +def from_container(container: 'EdgeContainer') -> 'RoutingDispatcher': """Create related RoutingDispatcher obj from config object""" if type(container) is not EdgeContainer: @@ -80,17 +115,18 @@ def from_container(container:'EdgeContainer') -> RoutingDispatcher: if not container.app_config.is_multi(): __print_splash(False) return container.dispatcher() - + + def __print_splash(in_multi_mode: bool): print(f''' -______ _ _____ _ -| ___ \\ (_) | ___| | | -| |_/ / __ _ ___ _ ___ ___ ___ _ __ ___ | |__ __| | __ _ ___ +______ _ _____ _ +| ___ \\ (_) | ___| | | +| |_/ / __ _ ___ _ ___ ___ ___ _ __ ___ | |__ __| | __ _ ___ | ___ \\/ _` / __| / __|/ __/ _ \\| '__/ _ \\ | __|/ _` |/ _` |/ _ \\ | |_/ / (_| \\__ \\ \\__ \\ (_| (_) | | | __/ | |__| (_| | (_| | __/ \\____/ \\__,_|___/_|___/\\___\\___/|_| \\___| \\____/\\__,_|\\__, |\\___| - __/ | - |___/ + __/ | + |___/ *********************************** Basiscore Edge diff --git a/bclib/edge_container.py b/bclib/edge_container.py index 3724eac..cab6ea5 100644 --- a/bclib/edge_container.py +++ b/bclib/edge_container.py @@ -1,52 +1,63 @@ import asyncio import sys -from dependency_injector import containers,providers -from bclib.logger import LoggerFactory +from dependency_injector import containers, providers +from bclib.logger.logger_factory import LoggerFactory from bclib.db_manager import DbManager -from bclib.cache import CacheFactory +from bclib.cache.factory import CacheFactory from bclib.utility import DictEx -from bclib.dispatcher import SocketDispatcher, DevServerDispatcher, EndpointDispatcher +from bclib.dispatcher.socket_dispatcher import SocketDispatcher +from bclib.dispatcher.dev_server_dispatcher import DevServerDispatcher +from bclib.dispatcher.endpoint_dispatcher import EndpointDispatcher from bclib.context.context_factory import ContextFactory -def get_mode(options:'DictEx'): + +def get_mode(options: 'DictEx'): if options.has("server"): ret_val = 'server' elif options.has("endpoint"): ret_val = 'endpoint' else: ret_val = 'socket' - return ret_val + return ret_val + -def create_loop(options:'DictEx') -> asyncio.AbstractEventLoop: - loop:asyncio.AbstractEventLoop = options.loop +def create_loop(options: 'DictEx') -> asyncio.AbstractEventLoop: + loop: asyncio.AbstractEventLoop = options.loop if loop is None and sys.platform == 'win32': - # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. - # Use Windows version of proactor event loop using IOCP - loop = asyncio.ProactorEventLoop() + # By default Windows can use only 64 sockets in asyncio loop. This is a limitation of underlying select() API call. + # Use Windows version of proactor event loop using IOCP + loop = asyncio.ProactorEventLoop() current_loop = asyncio.get_event_loop() if loop is not None and current_loop != loop: asyncio.set_event_loop(loop) return asyncio.get_event_loop() + class EdgeContainer(containers.DeclarativeContainer): app_config = providers.Configuration() app_container = providers.Object(providers.Self()) - app_options = providers.Singleton( DictEx,app_config) - app_mode = providers.Singleton( get_mode,app_options) - app_cache_options =providers.Singleton(lambda x:x.cache,app_options) - app_event_loop = providers.Singleton(create_loop,app_options) - app_cache_manager = providers.Singleton(CacheFactory.create,app_cache_options) - app_db_manager = providers.Singleton(DbManager,app_options, app_event_loop) - app_logger= providers.Singleton(LoggerFactory.create,app_options) - app_context_factory = providers.Singleton(ContextFactory,app_options,app_logger) - app_server_dispatcher = providers.Singleton(DevServerDispatcher,app_container,app_context_factory, app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) - app_endpoint_dispatcher = providers.Singleton(EndpointDispatcher,app_container,app_context_factory,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) - app_socket_dispatcher = providers.Singleton(SocketDispatcher,app_container,app_context_factory,app_options,app_cache_manager,app_db_manager,app_logger,app_event_loop) + app_options = providers.Singleton(DictEx, app_config) + app_mode = providers.Singleton(get_mode, app_options) + app_cache_options = providers.Singleton(lambda x: x.cache, app_options) + app_event_loop = providers.Singleton(create_loop, app_options) + app_cache_manager = providers.Singleton( + CacheFactory.create, app_cache_options) + app_db_manager = providers.Singleton( + DbManager, app_options, app_event_loop) + app_logger = providers.Singleton(LoggerFactory.create, app_options) + app_context_factory = providers.Singleton( + ContextFactory, app_options, app_logger) + app_server_dispatcher = providers.Singleton( + DevServerDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) + app_endpoint_dispatcher = providers.Singleton( + EndpointDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) + app_socket_dispatcher = providers.Singleton( + SocketDispatcher, app_container, app_context_factory, app_options, app_cache_manager, app_db_manager, app_logger, app_event_loop) dispatcher = providers.Selector( app_mode, server=app_server_dispatcher, - endpoint = app_endpoint_dispatcher, - socket = app_socket_dispatcher - ) \ No newline at end of file + endpoint=app_endpoint_dispatcher, + socket=app_socket_dispatcher + ) diff --git a/bclib/exception/__init__.py b/bclib/exception/__init__.py index 907ffb7..f345a8f 100644 --- a/bclib/exception/__init__.py +++ b/bclib/exception/__init__.py @@ -4,4 +4,4 @@ from bclib.exception.not_found_err import NotFoundErr from bclib.exception.handler_not_found_err import HandlerNotFoundErr from bclib.exception.bad_request_err import BadRequestErr -from bclib.exception.forbidden_err import ForbiddenErr \ No newline at end of file +from bclib.exception.forbidden_err import ForbiddenErr diff --git a/bclib/exception/bad_request_err.py b/bclib/exception/bad_request_err.py index 053b714..cb470df 100644 --- a/bclib/exception/bad_request_err.py +++ b/bclib/exception/bad_request_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from .short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class BadRequestErr(ShortCircuitErr): diff --git a/bclib/exception/forbidden_err.py b/bclib/exception/forbidden_err.py index b6705a2..cbe81c5 100644 --- a/bclib/exception/forbidden_err.py +++ b/bclib/exception/forbidden_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from .short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class ForbiddenErr(ShortCircuitErr): diff --git a/bclib/exception/handler_not_found_err.py b/bclib/exception/handler_not_found_err.py index 2bcad3c..03ff470 100644 --- a/bclib/exception/handler_not_found_err.py +++ b/bclib/exception/handler_not_found_err.py @@ -1,4 +1,4 @@ -from ..exception.not_found_err import NotFoundErr +from bclib.exception.not_found_err import NotFoundErr class HandlerNotFoundErr(NotFoundErr): diff --git a/bclib/exception/internal_server_err.py b/bclib/exception/internal_server_err.py index 2ed8d20..b3d97d8 100644 --- a/bclib/exception/internal_server_err.py +++ b/bclib/exception/internal_server_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class InternalServerErr(ShortCircuitErr): diff --git a/bclib/exception/not_found_err.py b/bclib/exception/not_found_err.py index 6363f98..e5580d5 100644 --- a/bclib/exception/not_found_err.py +++ b/bclib/exception/not_found_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class NotFoundErr(ShortCircuitErr): diff --git a/bclib/exception/unauthorized_err.py b/bclib/exception/unauthorized_err.py index 8a9e157..21cdff5 100644 --- a/bclib/exception/unauthorized_err.py +++ b/bclib/exception/unauthorized_err.py @@ -1,5 +1,5 @@ from bclib.utility.http_status_codes import HttpStatusCodes -from ..exception.short_circuit_err import ShortCircuitErr +from bclib.exception.short_circuit_err import ShortCircuitErr class UnauthorizedErr(ShortCircuitErr): diff --git a/bclib/listener/__init__.py b/bclib/listener/__init__.py index 63f273c..e69de29 100644 --- a/bclib/listener/__init__.py +++ b/bclib/listener/__init__.py @@ -1,11 +0,0 @@ -from bclib.listener.endpoint import Endpoint -from bclib.listener.socket_listener import SocketListener -from bclib.listener.rabbit_bus_listener import RabbitBusListener -from bclib.listener.message import Message -from bclib.listener.message_type import MessageType -from bclib.listener.http_listener.http_listener import HttpListener -from bclib.listener.http_listener.http_base_data_name import HttpBaseDataName -from bclib.listener.http_listener.http_base_data_type import HttpBaseDataType -from bclib.listener.web_message import WebMessage -from bclib.listener.socket_message import SocketMessage -from bclib.listener.end_point_message import EndPointMessage diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 327c8c5..894cbde 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -1,15 +1,16 @@ import asyncio import ssl from typing import Callable, TYPE_CHECKING, Coroutine, Optional -from ..endpoint import Endpoint -from bclib.utility import DictEx -from ..web_message import WebMessage +from aiohttp import web +from bclib.utility.dict_ex import DictEx if TYPE_CHECKING: - from aiohttp import web + from bclib.listener.endpoint import Endpoint + from bclib.listener.web_message import WebMessage from aiohttp.log import web_logger + class HttpListener: _id = 0 LOGGER = "logger" @@ -23,25 +24,29 @@ class HttpListener: _DEFAULT_MIDDLEWARES = () _DEFAULT_HANDLER_ARGS = None _DEFAULT_CLIENT_MAX_SIZE = 1024 ** 2 - - def __init__(self, endpoint: Endpoint, async_callback: 'Callable[[WebMessage],Coroutine]',ssl_options:'dict', configuration: Optional[DictEx]): + def __init__(self, endpoint: 'Endpoint', async_callback: 'Callable[[WebMessage],Coroutine]', ssl_options: 'dict', configuration: 'Optional[DictEx]'): self.__endpoint = endpoint self.on_message_receive_async = async_callback self.ssl_options = ssl_options self.__config = configuration if configuration is not None else DictEx() - self.__logger = self.__config.get(HttpListener.LOGGER, HttpListener._DEFAULT_LOGGER) - self.__router = self.__config.get(HttpListener.ROUTER, HttpListener._DEFAULT_ROUTER) - self.__middlewares = self.__config.get(HttpListener.MIDDLEWARES, HttpListener._DEFAULT_MIDDLEWARES) - self.__handler_args = self.__config.get(HttpListener.HANDLER_ARGS, HttpListener._DEFAULT_HANDLER_ARGS) - self.__client_max_size = self.__config.get(HttpListener.CLIENT_MAX_SIZE, HttpListener._DEFAULT_CLIENT_MAX_SIZE) - + self.__logger = self.__config.get( + HttpListener.LOGGER, HttpListener._DEFAULT_LOGGER) + self.__router = self.__config.get( + HttpListener.ROUTER, HttpListener._DEFAULT_ROUTER) + self.__middlewares = self.__config.get( + HttpListener.MIDDLEWARES, HttpListener._DEFAULT_MIDDLEWARES) + self.__handler_args = self.__config.get( + HttpListener.HANDLER_ARGS, HttpListener._DEFAULT_HANDLER_ARGS) + self.__client_max_size = self.__config.get( + HttpListener.CLIENT_MAX_SIZE, HttpListener._DEFAULT_CLIENT_MAX_SIZE) def initialize_task(self, event_loop: asyncio.AbstractEventLoop): event_loop.create_task(self.__server_task(event_loop)) async def __server_task(self, event_loop: asyncio.AbstractEventLoop): from aiohttp import web + async def on_request_receive_async(request: 'web.Request') -> web.Response: msg = WebMessage(request) await self.on_message_receive_async(msg) @@ -57,7 +62,7 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: ) app.add_routes( [web.route('*', '/{tail:.*}', on_request_receive_async)]) - ssl_context= None + ssl_context = None if self.ssl_options: ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) if 'certfile' in self.ssl_options: @@ -73,11 +78,13 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: ) ssl_context.load_cert_chain(certfile=pem_file_path) except Exception as e: - raise Exception("Invalid PKCS12 or pastphrase for {0}: {1}".format(self.ssl_options.pfxfile, e)) + raise Exception("Invalid PKCS12 or pastphrase for {0}: {1}".format( + self.ssl_options.pfxfile, e)) runner = web.AppRunner(app, handle_signals=True) await runner.setup() - site = web.TCPSite(runner, self.__endpoint.url, self.__endpoint.port, ssl_context=ssl_context) + site = web.TCPSite(runner, self.__endpoint.url, + self.__endpoint.port, ssl_context=ssl_context) await site.start() print( f"Development Edge server started at http{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port}") @@ -87,23 +94,27 @@ async def on_request_receive_async(request: 'web.Request') -> web.Response: except asyncio.CancelledError: pass finally: - print(f"Development Edge server for http{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port} stopped.") + print(f"Development Edge server for http" + + f"{'s' if self.ssl_options else ''}://{self.__endpoint.url}:{self.__endpoint.port} stopped.") await site.stop() await runner.cleanup() await runner.shutdown() @staticmethod - def convert_pfx_to_pem_file(pfxfile:str,password:str)->str: + def convert_pfx_to_pem_file(pfxfile: str, password: str) -> str: from cryptography.hazmat.primitives.serialization import pkcs12, Encoding, PrivateFormat, NoEncryption - with open(pfxfile,"rb") as f: + with open(pfxfile, "rb") as f: try: - private_key, certificate, additional_certificates = pkcs12.load_key_and_certificates(f.read(), password.encode()) + private_key, certificate, additional_certificates = pkcs12.load_key_and_certificates( + f.read(), password.encode()) pem_file_path = '{0}.auto-generated.pem'.format(pfxfile) with open(pem_file_path, 'wb') as pem_file: pem_file.write(certificate.public_bytes(Encoding.PEM)) for item in additional_certificates: pem_file.write(item.public_bytes(Encoding.PEM)) - pem_file.write(private_key.private_bytes(encoding= Encoding.PEM,format=PrivateFormat.TraditionalOpenSSL,encryption_algorithm=NoEncryption())) + pem_file.write(private_key.private_bytes( + encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, encryption_algorithm=NoEncryption())) return pem_file_path except Exception as ex: - raise Exception("Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) \ No newline at end of file + raise Exception( + "Error in create pem file from pfx {0}: {1}".format(pfxfile, ex)) diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index 17bb8d1..88104bf 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -1,8 +1,8 @@ from struct import error -from bclib.context.rabbit_context import RabbitContext from typing import TYPE_CHECKING +from bclib.context.rabbit_context import RabbitContext from bclib.listener.rabbit_listener import RabbitListener -from bclib.utility import DictEx +from bclib.utility.dict_ex import DictEx if TYPE_CHECKING: from bclib.dispatcher import IDispatcher diff --git a/bclib/listener/stream_base_message.py b/bclib/listener/stream_base_message.py index 86e04ec..a7cfaf8 100644 --- a/bclib/listener/stream_base_message.py +++ b/bclib/listener/stream_base_message.py @@ -3,16 +3,16 @@ from typing import Any, Coroutine, Optional from bclib.listener.message_type import MessageType -from .message import Message +from bclib.listener.message import Message class StreamBaseMessage(Message): - def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + def __init__(self, reader: 'asyncio.StreamReader', writer: 'asyncio.StreamWriter'): self.reader = reader self.writer = writer - self.buffer:Optional[bytes] = None + self.buffer: Optional[bytes] = None - async def get_json_async(self)-> Coroutine[Any, Any, dict]: + async def get_json_async(self) -> Coroutine[Any, Any, dict]: await self._fill_async() return json.loads(self.buffer) @@ -33,7 +33,7 @@ async def _fill_async(self) -> Coroutine[Any, Any, None]: data = await self.reader.readexactly(data_len) self.buffer = data - async def write_result_async(self, cms: dict,message_type:'MessageType'): + async def write_result_async(self, cms: dict, message_type: 'MessageType'): try: self.writer.write(message_type.value.to_bytes(1, 'big')) data = self.session_id.encode() @@ -51,7 +51,4 @@ async def write_result_async(self, cms: dict,message_type:'MessageType'): pass async def set_result_async(self, cms: dict): - await self.write_result_async(cms,self.type) - - - + await self.write_result_async(cms, self.type) diff --git a/bclib/logger/__init__.py b/bclib/logger/__init__.py index 7403b8b..e69de29 100644 --- a/bclib/logger/__init__.py +++ b/bclib/logger/__init__.py @@ -1,3 +0,0 @@ -from bclib.logger.ilogger import ILogger -from bclib.logger.logger_factory import LoggerFactory -from bclib.logger.log_object import LogObject \ No newline at end of file diff --git a/bclib/logger/ilogger.py b/bclib/logger/ilogger.py index b1526e8..abca7a1 100644 --- a/bclib/logger/ilogger.py +++ b/bclib/logger/ilogger.py @@ -3,12 +3,13 @@ import traceback from typing import Optional from bclib.utility import DictEx -from .log_object import LogObject +from bclib.logger.log_object import LogObject + class ILogger(ABC): """Base class for logger""" - def __init__(self,options:'DictEx'): + def __init__(self, options: 'DictEx'): self.options = options self.name = options["name"] if options.has("name") else None self.__log_name = f"{self.name}: " if self.name else '' @@ -17,7 +18,6 @@ def __init__(self,options:'DictEx'): self.__log_request: bool = self.options.log_request if self.options.has( "log_request") else True print(f'{self.__class__.__name__} start logging') - @abstractmethod async def log_async(self, log_object: LogObject): @@ -26,12 +26,11 @@ async def log_async(self, log_object: LogObject): def new_object_log(self, schema_name: str, routing_key: Optional[str] = None, **kwargs) -> LogObject: """New object log""" return LogObject(schema_name, routing_key, **kwargs) - - def log_request(self, message:'str'): - if(self.__log_request): - print(self.__log_name,'LOG', message) - def log_error(self,error:'Exception'): - print(self.__log_name,'ERROR',str(error)) - traceback.print_exc() + def log_request(self, message: 'str'): + if (self.__log_request): + print(self.__log_name, 'LOG', message) + def log_error(self, error: 'Exception'): + print(self.__log_name, 'ERROR', str(error)) + traceback.print_exc() diff --git a/bclib/logger/logger_factory.py b/bclib/logger/logger_factory.py index 596230d..b3ea0df 100644 --- a/bclib/logger/logger_factory.py +++ b/bclib/logger/logger_factory.py @@ -1,9 +1,9 @@ from typing import Optional from bclib.utility import DictEx -from .rabbit_schema_base_logger import RabbitSchemaBaseLogger -from .restful_schema_base_logger import RESTfulSchemaBaseLogger -from .no_logger import NoLogger -from .ilogger import ILogger +from bclib.logger.rabbit_schema_base_logger import RabbitSchemaBaseLogger +from bclib.logger.restful_schema_base_logger import RESTfulSchemaBaseLogger +from bclib.logger.no_logger import NoLogger +from bclib.logger.ilogger import ILogger class LoggerFactory: diff --git a/bclib/logger/no_logger.py b/bclib/logger/no_logger.py index 686ae20..3e236a3 100644 --- a/bclib/logger/no_logger.py +++ b/bclib/logger/no_logger.py @@ -1,13 +1,13 @@ from bclib.utility import DictEx -from .log_object import LogObject -from .ilogger import ILogger +from bclib.logger.log_object import LogObject +from bclib.logger.ilogger import ILogger class NoLogger(ILogger): """class for no logging""" - def __init__(self,options:'DictEx'): + def __init__(self, options: 'DictEx'): super().__init__(options) async def log_async(self, log_object: LogObject): - """log data async""" \ No newline at end of file + """log data async""" diff --git a/bclib/logger/rabbit_schema_base_logger.py b/bclib/logger/rabbit_schema_base_logger.py index 02ff568..0b0493f 100644 --- a/bclib/logger/rabbit_schema_base_logger.py +++ b/bclib/logger/rabbit_schema_base_logger.py @@ -1,7 +1,7 @@ import asyncio import json from typing import Optional -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx @@ -16,11 +16,13 @@ def __init__(self, options: 'DictEx') -> None: if "queue" in self.__connection_options and "exchange" in self.__connection_options: raise Exception("'queue' not acceptable when 'exchange' is set") elif "queue" not in self.__connection_options and "exchange" not in self.__connection_options: - raise Exception("'exchange' or 'queue' must be set in connection option") - + raise Exception( + "'exchange' or 'queue' must be set in connection option") + async def _save_schema_async(self, schema: dict, routing_key: Optional[str] = None): if routing_key is not None and self.__connection_options.queue is not None: - raise Exception("'routing key' is not acceptable when 'queue' is in options") + raise Exception( + "'routing key' is not acceptable when 'queue' is in options") def send_to_rabbit(): import pika @@ -40,8 +42,8 @@ def send_to_rabbit(): auto_delete=self.__connection_options.auto_delete or False ) channel.basic_publish( - exchange=self.__connection_options.exchange or "", - routing_key=routing_key or queue or "", + exchange=self.__connection_options.exchange or "", + routing_key=routing_key or queue or "", body=json.dumps(schema, ensure_ascii=False), properties=pika.BasicProperties( content_type="application/json", diff --git a/bclib/logger/restful_schema_base_logger.py b/bclib/logger/restful_schema_base_logger.py index 554f243..0a99073 100644 --- a/bclib/logger/restful_schema_base_logger.py +++ b/bclib/logger/restful_schema_base_logger.py @@ -1,4 +1,4 @@ -from ..logger.schema_base_logger import SchemaBaseLogger +from bclib.logger.schema_base_logger import SchemaBaseLogger from bclib.utility import DictEx diff --git a/bclib/logger/schema_base_logger.py b/bclib/logger/schema_base_logger.py index d4e2707..814d832 100644 --- a/bclib/logger/schema_base_logger.py +++ b/bclib/logger/schema_base_logger.py @@ -3,8 +3,8 @@ from bclib.logger.log_object import LogObject from bclib.utility import DictEx -from ..logger.log_schema import LogSchema -from ..logger.ilogger import ILogger +from bclib.logger.log_schema import LogSchema +from bclib.logger.ilogger import ILogger class SchemaBaseLogger(ILogger): diff --git a/bclib/parser/__init__.py b/bclib/parser/__init__.py index 9086f16..9974d41 100644 --- a/bclib/parser/__init__.py +++ b/bclib/parser/__init__.py @@ -3,5 +3,5 @@ from bclib.parser.answer import Answer, UserActionTypes, UserAction -def ParseAnswer(json: 'str|Any') -> Answer: +def ParseAnswer(json: 'str|Any') -> 'Answer': return Answer(json) diff --git a/bclib/parser/answer/answer.py b/bclib/parser/answer/answer.py index 8796130..d741fbd 100644 --- a/bclib/parser/answer/answer.py +++ b/bclib/parser/answer/answer.py @@ -5,23 +5,23 @@ from bclib.parser.answer.storage_data import StorageData from bclib.parser.answer.question_data import QuestionData from bclib import parser -from bclib.db_manager import RESTfulConnection +from bclib.db_manager.restful_connection import RESTfulConnection from bclib.parser.answer.validators import Validator from bclib.utility import DictEx -from ..answer.user_action_types import UserActionTypes -from ..answer.user_action import UserAction -import asyncio +from bclib.parser.answer.user_action_types import UserActionTypes +from bclib.parser.answer.user_action import UserAction + class Answer: """BasisJsonParser is a tool to parse basis_core components json objects. This tool is developed based on basis_core key and values.""" - def __init__(self, data: 'str|Any', api_url: 'str' = None, check_validation:"bool"= False): + def __init__(self, data: 'str|Any', api_url: 'str' = None, check_validation: "bool" = False): self.json = json.loads(data) if isinstance(data, str) else data self.__answer_list: 'list[UserAction]' = None self.__api_connection = RESTfulConnection( api_url) if api_url else None - self.check_validation = check_validation + self.check_validation = check_validation async def __fill_answer_list_async(self): self.__answer_list = list() @@ -32,13 +32,16 @@ async def __fill_answer_list_async(self): if action_type.value in list(data.keys()): prp_id = data['propId'] if action_type != UserActionTypes.ANSWERS else data["prpId"] for actions in data[action_type.value]: - prp_value_id = actions['id'] if 'id' in actions.keys() else None + prp_value_id = actions['id'] if 'id' in actions.keys( + ) else None if 'parts' in actions.keys(): for parts in actions['parts']: internal_prp_value_id = internal_prp_value_index - part_number = parts['part'] if "part" in parts.keys() else None + part_number = parts['part'] if "part" in parts.keys( + ) else None for values in parts['values']: - value_id = values['id'] if "id" in values.keys() else None + value_id = values['id'] if "id" in values.keys( + ) else None value = values['value'] answer = parser.ParseAnswer( values["answer"]) if 'answer' in values.keys() else None @@ -70,7 +73,8 @@ async def __enrich_data_async(self): for validations in parts.parts ] questions_info.append( - QuestionData(parts.prpId, parts.OwnerID, parts.TypeID if "TypeID" in parts else parts.typeid, parts.wordId, enriched_data_list) + QuestionData( + parts.prpId, parts.OwnerID, parts.TypeID if "TypeID" in parts else parts.typeid, parts.wordId, enriched_data_list) ) if len(questions_info) > 0: for values in self.__answer_list: @@ -88,23 +92,27 @@ async def __enrich_data_async(self): values.table = storage_data.table values.field = storage_data.field if self.check_validation and values.action != UserActionTypes.DELETED: - status, message = Validator.check_validators(enriched_data.validators, values.value) + status, message = Validator.check_validators( + enriched_data.validators, values.value) values.validation_status = status values.validation_message = message - - def __enrich_data(self, validations:DictEx) -> 'EnrichedData': + + def __enrich_data(self, validations: DictEx) -> 'EnrichedData': part_id = validations.part data_type = self.__set_data_type(validations) val_val = validations.validations - storage_data = self.__set_storage_data(val_val) if isinstance(val_val, dict) else None - validators = val_val if self.check_validation and isinstance(validations.validations, dict) else {} + storage_data = self.__set_storage_data( + val_val) if isinstance(val_val, dict) else None + validators = val_val if self.check_validation and isinstance( + validations.validations, dict) else {} return EnrichedData(part_id, data_type, storage_data, validators) - def __set_data_type(self, validations:DictEx) -> 'str': + def __set_data_type(self, validations: DictEx) -> 'str': has_link = True if validations.link else False val_val = validations.validations - data_type = val_val["dataType"] if isinstance(val_val, dict) and "dataType" in val_val else None - + data_type = val_val["dataType"] if isinstance( + val_val, dict) and "dataType" in val_val else None + return self.__data_type_checker(validations.viewType, data_type, has_link) def __data_type_checker(self, view_type: str, datatype: str = None, has_link: bool = None): @@ -137,13 +145,13 @@ def __data_type_checker(self, view_type: str, datatype: str = None, has_link: bo else: result = "None" return result - - def __set_storage_data(self, val_val:"dict") -> "StorageData": + + def __set_storage_data(self, val_val: "dict") -> "StorageData": database = val_val["database"] if "database" in val_val else None table = val_val["table"] if "table" in val_val else None field = val_val["field"] if "field" in val_val else None return StorageData(database, table, field) - + async def __get_action_async(self, prp_id_list: 'list[int]', action_list: 'list[UserActionTypes]', part_list: 'list[int]', is_file: "bool" = None, predicate: 'Callable[[UserAction],bool]' = None) -> 'list[UserAction]': ret_val: 'list[UserAction]' = None if self.__answer_list is None: diff --git a/bclib/parser/answer/question_data.py b/bclib/parser/answer/question_data.py index ec752b5..51ced11 100644 --- a/bclib/parser/answer/question_data.py +++ b/bclib/parser/answer/question_data.py @@ -1,9 +1,10 @@ -from ..answer.enriched_data import EnrichedData +from bclib.parser.answer.enriched_data import EnrichedData + class QuestionData: - def __init__(self, prpid:"int", ownerid:"int|None", typeid:"int|None", wordid:"int|None", enriched_data:"list[EnrichedData]") -> None: + def __init__(self, prpid: "int", ownerid: "int|None", typeid: "int|None", wordid: "int|None", enriched_data: "list[EnrichedData]") -> None: self.prpid = prpid self.ownerid = ownerid if ownerid is not None else 0 self.typeid = typeid self.wordid = wordid - self.enriched_data = enriched_data \ No newline at end of file + self.enriched_data = enriched_data diff --git a/bclib/parser/answer/user_action.py b/bclib/parser/answer/user_action.py index 4dc9880..bf12e45 100644 --- a/bclib/parser/answer/user_action.py +++ b/bclib/parser/answer/user_action.py @@ -1,10 +1,10 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from bclib.parser.answer.answer import Answer -from ..answer.user_action_types import UserActionTypes -from ..answer.file_user_action import FileUserAction -from ..answer.date_user_action import DateUserAction -from ..answer.time_user_action import TimeUserAction +from bclib.parser.answer.user_action_types import UserActionTypes +from bclib.parser.answer.file_user_action import FileUserAction +from bclib.parser.answer.date_user_action import DateUserAction +from bclib.parser.answer.time_user_action import TimeUserAction class UserAction: @@ -22,19 +22,19 @@ def __init__(self, prp_id: 'int', action: 'UserActionTypes', prp_value_id: 'int' self.multi = multi self.answer: 'Answer' = answer self.internal_prp_value_id: "int" = internal_prp_value_id - self.database:"str" = None - self.table:"str" = None - self.field:"str" = None - self.ownerid:"int" = 0 - self.typeid:"int" = None - self.wordid:"int" = None + self.database: "str" = None + self.table: "str" = None + self.field: "str" = None + self.ownerid: "int" = 0 + self.typeid: "int" = None + self.wordid: "int" = None self.validation_status: "bool" = True self.validation_message: "list[str]" = [] def as_tuple(self) -> tuple: return ( self.prp_id, self.action.value, self.prp_value_id, self.internal_prp_value_id, self.value_id, - self.value, self.part, self.datatype, self.database, self.table, self.field, self.multi, + self.value, self.part, self.datatype, self.database, self.table, self.field, self.multi, self.ownerid, self.typeid, self.wordid, self.answer, self.validation_status, self.validation_message ) @@ -62,7 +62,7 @@ def as_dict(self) -> dict: def is_date_useraction(self): return self.datatype == "datevalue" - + def as_date_useraction(self): if self.is_date_useraction(): value = self.value @@ -88,7 +88,7 @@ def as_time_useraction(self): value["time"], int(value["timeid"]) ) - + def is_file_content(self): return self.datatype == "files" diff --git a/bclib/parser/html/html_parser_ex.py b/bclib/parser/html/html_parser_ex.py index 610a4dc..2c8758e 100644 --- a/bclib/parser/html/html_parser_ex.py +++ b/bclib/parser/html/html_parser_ex.py @@ -2,7 +2,7 @@ from html.parser import HTMLParser from typing import Any from bclib.utility import DictEx -from ..html.html_tag import HtmlTag +from bclib.parser.html.html_tag import HtmlTag class HtmlParserEx(HTMLParser): diff --git a/bclib/predicate/all.py b/bclib/predicate/all.py index 782004b..71d77e8 100644 --- a/bclib/predicate/all.py +++ b/bclib/predicate/all.py @@ -1,5 +1,7 @@ -from ..predicate.predicate import Predicate -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class All (Predicate): @@ -9,7 +11,7 @@ def __init__(self, *predicate: 'Predicate') -> None: super().__init__(None) self.__predicate_list = predicate - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': is_ok = True try: for predicate in self.__predicate_list: diff --git a/bclib/predicate/any.py b/bclib/predicate/any.py index c8651f7..5c22b61 100644 --- a/bclib/predicate/any.py +++ b/bclib/predicate/any.py @@ -1,5 +1,7 @@ -from ..predicate.predicate import Predicate -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Any (Predicate): @@ -9,7 +11,7 @@ def __init__(self, *predicate: 'Predicate') -> None: super().__init__(None) self.__predicate_list = predicate - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': is_ok = False try: for predicate in self.__predicate_list: diff --git a/bclib/predicate/between.py b/bclib/predicate/between.py index ddbe5a2..8a3ab00 100644 --- a/bclib/predicate/between.py +++ b/bclib/predicate/between.py @@ -1,5 +1,7 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Between(Predicate): @@ -10,9 +12,9 @@ def __init__(self, expression: str, min_value: int, max_value: int): self.__min_value = min_value self.__max_value = max_value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__min_value < int(value) < self.__max_value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/callback.py b/bclib/predicate/callback.py index 42d08c5..3bd79c4 100644 --- a/bclib/predicate/callback.py +++ b/bclib/predicate/callback.py @@ -1,17 +1,19 @@ from bclib.exception import ShortCircuitErr -from ..predicate.predicate import Predicate -from typing import Callable, Coroutine -from bclib.context import Context +from bclib.predicate.predicate import Predicate +from typing import Callable, Coroutine, TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Callback (Predicate): - """Create callback base cheking predicate""" + """Create callback base checking predicate""" def __init__(self, callback: 'Callable[[Context],Coroutine[bool]]') -> None: super().__init__(None) self.__callback = callback - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: return await self.__callback(context) except ShortCircuitErr: diff --git a/bclib/predicate/equal.py b/bclib/predicate/equal.py index 9f462ed..3db596b 100644 --- a/bclib/predicate/equal.py +++ b/bclib/predicate/equal.py @@ -1,5 +1,7 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class Equal (Predicate): @@ -9,9 +11,9 @@ def __init__(self, expression, value) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value == value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/greater_than.py b/bclib/predicate/greater_than.py index 8921fa6..ef9b7c5 100644 --- a/bclib/predicate/greater_than.py +++ b/bclib/predicate/greater_than.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class GreaterThan (Predicate): - """Create greater than cheking predicate""" + """Create greater than checking predicate""" def __init__(self, expression: str, value: int) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value < value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/greater_than_equal.py b/bclib/predicate/greater_than_equal.py index 797d1db..e5c2216 100644 --- a/bclib/predicate/greater_than_equal.py +++ b/bclib/predicate/greater_than_equal.py @@ -1,17 +1,19 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from bclib.context.context import Context class GreaterThanEqual (Predicate): - """Create greater than equal cheking predicate""" + """Create greater than equal checking predicate""" def __init__(self, expression: str, value: int) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value <= value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/has_value.py b/bclib/predicate/has_value.py index c93ebb6..d963e9b 100644 --- a/bclib/predicate/has_value.py +++ b/bclib/predicate/has_value.py @@ -1,17 +1,19 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate -from typing import Any +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class HasValue (Predicate): - """Create has value cheking predicate""" + """Create has value checking predicate""" def __init__(self, expression: str) -> None: super().__init__(expression) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return False if not value or value.isspace() else True except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/in_list.py b/bclib/predicate/in_list.py index 9e799b2..9af8895 100644 --- a/bclib/predicate/in_list.py +++ b/bclib/predicate/in_list.py @@ -1,18 +1,20 @@ -from typing import Any -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from bclib.context.context import Context class InList(Predicate): - """Create list cheking predicate""" + """Create list checking predicate""" def __init__(self, expression: str, *items: Any) -> None: super().__init__(expression) self.__items = [*items] - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return value in self.__items except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/less_than.py b/bclib/predicate/less_than.py index 179daf3..d3937a7 100644 --- a/bclib/predicate/less_than.py +++ b/bclib/predicate/less_than.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class LessThan (Predicate): - """Create less than cheking predicate""" + """Create less than checking predicate""" - def __init__(self, expression: str, value: int) -> None: + def __init__(self, expression: 'str', value: 'int') -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value > value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/less_than_equal.py b/bclib/predicate/less_than_equal.py index 0edde36..c8d9345 100644 --- a/bclib/predicate/less_than_equal.py +++ b/bclib/predicate/less_than_equal.py @@ -1,17 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class LessThanEqual (Predicate): - """Create less than and equal cheking predicate""" + """Create less than and equal checking predicate""" - def __init__(self, expression: str, value: int) -> None: + def __init__(self, expression: 'str', value: 'int') -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value >= value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/match.py b/bclib/predicate/match.py index 4c22a6a..ca552c1 100644 --- a/bclib/predicate/match.py +++ b/bclib/predicate/match.py @@ -1,18 +1,21 @@ import re -from bclib.context import Context -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Match (Predicate): - """Create regex matching cheking predicate""" + """Create regex matching checking predicate""" - def __init__(self, expression: str, value: str) -> None: + def __init__(self, expression: 'str', value: 'str') -> None: super().__init__(expression) self.__compiled_regex = re.compile(value) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__compiled_regex.match(str(value)) != None except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/not_equal.py b/bclib/predicate/not_equal.py index 46e67a7..28cd371 100644 --- a/bclib/predicate/not_equal.py +++ b/bclib/predicate/not_equal.py @@ -1,18 +1,20 @@ -from bclib.context import Context -from ..predicate.predicate import Predicate -from typing import Any +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from bclib.context.context import Context class NotEqual (Predicate): - """Create not equality cheking predicate""" + """Create not equality checking predicate""" def __init__(self, expression: str, value: Any) -> None: super().__init__(expression) self.__value = value - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: - value = eval(self.exprossion, {}, {"context": context}) + value = eval(self.expression, {}, {"context": context}) return self.__value != value except: # pylint: disable=bare-except return False diff --git a/bclib/predicate/predicate.py b/bclib/predicate/predicate.py index 24eadfc..326877e 100644 --- a/bclib/predicate/predicate.py +++ b/bclib/predicate/predicate.py @@ -1,14 +1,17 @@ from abc import ABC, abstractmethod -from bclib.context import Context +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Predicate(ABC): """Base class for predicate""" - def __init__(self, expression: str) -> None: + def __init__(self, expression: 'str') -> 'None': super().__init__() - self.exprossion = expression + self.expression = expression @abstractmethod - async def check_async(self, context: Context) -> bool: - """Applay cheking for predicate""" + async def check_async(self, context: 'Context') -> 'bool': + """Apply checking for predicate""" diff --git a/bclib/predicate/url.py b/bclib/predicate/url.py index 96f51ba..57356a9 100644 --- a/bclib/predicate/url.py +++ b/bclib/predicate/url.py @@ -1,17 +1,20 @@ from types import FunctionType -from bclib.context import Context from bclib.utility import DictEx -from ..predicate.predicate import Predicate +from bclib.predicate.predicate import Predicate +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from bclib.context.context import Context class Url (Predicate): - """Create Url cheking predicate""" + """Create Url checking predicate""" - def __init__(self, expression: str) -> None: + def __init__(self, expression: 'str') -> 'None': super().__init__(expression) self.__validator: FunctionType = Url.__generate_validator(expression) - async def check_async(self, context: Context) -> bool: + async def check_async(self, context: 'Context') -> 'bool': try: is_ok, url_parts = self.__validator(context.url) if is_ok and url_parts: @@ -60,7 +63,7 @@ def url_function(url): try: url_parts = url.split("/") if {" and ".join(where_part_list)}: - {','.join(segment_list)} = url_parts{"[0]" if len(segment_list)==1 else ""} + {','.join(segment_list)} = url_parts{"[0]" if len(segment_list) == 1 else ""} return (True,{{ {','.join(return_dict_property_names)} }}) else: return (False,None) From 3de4f21ef830072ae6160d812b0f3b20780ca2d1 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:10:17 +0330 Subject: [PATCH 10/15] edit --- bclib/dispatcher/dev_server_dispatcher.py | 4 ++-- bclib/listener/http_listener/http_listener.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bclib/dispatcher/dev_server_dispatcher.py b/bclib/dispatcher/dev_server_dispatcher.py index 3b01719..82560fb 100644 --- a/bclib/dispatcher/dev_server_dispatcher.py +++ b/bclib/dispatcher/dev_server_dispatcher.py @@ -1,10 +1,10 @@ import asyncio -from context.context_factory import ContextFactory from dependency_injector import containers +from bclib.context.context_factory import ContextFactory from bclib.logger.ilogger import ILogger -from cache.cache_manager import CacheManager +from bclib.cache.cache_manager import CacheManager from bclib.db_manager import DbManager from bclib.utility import DictEx from bclib.listener.endpoint import Endpoint diff --git a/bclib/listener/http_listener/http_listener.py b/bclib/listener/http_listener/http_listener.py index 894cbde..a539f32 100644 --- a/bclib/listener/http_listener/http_listener.py +++ b/bclib/listener/http_listener/http_listener.py @@ -2,13 +2,12 @@ import ssl from typing import Callable, TYPE_CHECKING, Coroutine, Optional from aiohttp import web +from aiohttp.log import web_logger from bclib.utility.dict_ex import DictEx +from bclib.listener.web_message import WebMessage if TYPE_CHECKING: from bclib.listener.endpoint import Endpoint - from bclib.listener.web_message import WebMessage - -from aiohttp.log import web_logger class HttpListener: From d3867696fa7947491be4a7ecafd143d2a1f9bdc5 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:26:33 +0330 Subject: [PATCH 11/15] fix --- requirements.tx | 4 ++++ setup.py | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 requirements.tx diff --git a/requirements.tx b/requirements.tx new file mode 100644 index 0000000..d1dd406 --- /dev/null +++ b/requirements.tx @@ -0,0 +1,4 @@ +aiohttp==3.11.10 +dependency-injector==4.44.0 +legacy-cgi==2.6.1 +pika==1.3.2 diff --git a/setup.py b/setup.py index 4c212c5..681f883 100644 --- a/setup.py +++ b/setup.py @@ -21,13 +21,13 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - # install_requires=[ - # 'pika', - # 'requests', - # 'pymongo', - # 'pyodbc' - # ], - #package_dir={"": "basiscore"}, + install_requires=[ + 'pika', + 'aiohttp', + 'legacy-cgi', + 'dependency-injector' + ], + # package_dir={"": "basiscore"}, packages=setuptools.find_packages(exclude=["test", "app-env", ".vscode"]), python_requires=">=3.7", setup_requires=['wheel'] From 43ac46b2933bea20e2c7bd85d5c5cf8210426050 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:40:04 +0330 Subject: [PATCH 12/15] add --- requirements.tx | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.tx b/requirements.tx index d1dd406..0103311 100644 --- a/requirements.tx +++ b/requirements.tx @@ -1,4 +1,3 @@ aiohttp==3.11.10 dependency-injector==4.44.0 legacy-cgi==2.6.1 -pika==1.3.2 diff --git a/setup.py b/setup.py index 681f883..e00f034 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ "Operating System :: OS Independent", ], install_requires=[ - 'pika', 'aiohttp', 'legacy-cgi', 'dependency-injector' From d288c0ec66a7f42e71005c2790adc04e6aad1f11 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:39:52 +0330 Subject: [PATCH 13/15] fix bug --- bclib/listener/rabbit_bus_listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bclib/listener/rabbit_bus_listener.py b/bclib/listener/rabbit_bus_listener.py index 88104bf..6470063 100644 --- a/bclib/listener/rabbit_bus_listener.py +++ b/bclib/listener/rabbit_bus_listener.py @@ -5,7 +5,7 @@ from bclib.utility.dict_ex import DictEx if TYPE_CHECKING: - from bclib.dispatcher import IDispatcher + from bclib.dispatcher.idispatcher import IDispatcher class RabbitBusListener(RabbitListener): From 9473bfab399c82207148465244429bf46e7fcbf2 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:53:22 +0330 Subject: [PATCH 14/15] fix import --- bclib/dispatcher/dispatcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bclib/dispatcher/dispatcher.py b/bclib/dispatcher/dispatcher.py index 8d1c365..b9a67e8 100644 --- a/bclib/dispatcher/dispatcher.py +++ b/bclib/dispatcher/dispatcher.py @@ -6,6 +6,7 @@ from typing import Awaitable, Callable, Any, Coroutine, Optional, TYPE_CHECKING from functools import wraps from dependency_injector import containers +from listener.rabbit_bus_listener import RabbitBusListener from bclib.logger.ilogger import ILogger from cache.cache_manager import CacheManager From b9176b1976a4803a80b25ea6fcb2af2e3729fde5 Mon Sep 17 00:00:00 2001 From: Qamsari <44198226+Qamsari@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:59:21 +0330 Subject: [PATCH 15/15] edit version --- bclib/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bclib/__init__.py b/bclib/__init__.py index d04a09b..00a02cb 100644 --- a/bclib/__init__.py +++ b/bclib/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.34.1" +__version__ = "3.35.0" import bclib.context diff --git a/setup.py b/setup.py index e00f034..327d744 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,6 @@ ], # package_dir={"": "basiscore"}, packages=setuptools.find_packages(exclude=["test", "app-env", ".vscode"]), - python_requires=">=3.7", + python_requires=">=3.13", setup_requires=['wheel'] )