From 9760f5c83e3893da0edefd6a0ce12a13405ee0c3 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Wed, 12 Nov 2025 11:40:37 +0100 Subject: [PATCH 01/19] feat(external_worker): introduce ExternalWorkerConfig for improved configuration management and update FuncNodesExternalWorker to utilize it --- src/funcnodes_worker/__init__.py | 3 +- src/funcnodes_worker/external_worker.py | 44 ++++++++++++++++++++- src/funcnodes_worker/websocket.py | 4 +- src/funcnodes_worker/worker.py | 52 +++++++++++++++++++++++-- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/src/funcnodes_worker/__init__.py b/src/funcnodes_worker/__init__.py index 81eac0f..f619e0d 100644 --- a/src/funcnodes_worker/__init__.py +++ b/src/funcnodes_worker/__init__.py @@ -3,7 +3,7 @@ from .worker import Worker from .remote_worker import RemoteWorker -from .external_worker import FuncNodesExternalWorker +from .external_worker import FuncNodesExternalWorker, ExternalWorkerConfig from .loop import CustomLoop if not aiohttp: @@ -17,6 +17,7 @@ __all__ = [ + "ExternalWorkerConfig", "Worker", "RemoteWorker", "FuncNodesExternalWorker", diff --git a/src/funcnodes_worker/external_worker.py b/src/funcnodes_worker/external_worker.py index 4c0dfeb..b8051c7 100644 --- a/src/funcnodes_worker/external_worker.py +++ b/src/funcnodes_worker/external_worker.py @@ -1,8 +1,15 @@ from __future__ import annotations -from typing import Dict, List, TypedDict +from typing import Dict, List, TypedDict, Union, Any, Optional, Type from funcnodes_worker.loop import CustomLoop from funcnodes_core import NodeClassMixin, JSONEncoder, Encdata, EventEmitterMixin from weakref import WeakValueDictionary +from pydantic import BaseModel + + +class ExternalWorkerConfig(BaseModel): + """ + A class that represents the configuration of an external worker. + """ class FuncNodesExternalWorker(NodeClassMixin, EventEmitterMixin, CustomLoop): @@ -10,10 +17,16 @@ class FuncNodesExternalWorker(NodeClassMixin, EventEmitterMixin, CustomLoop): A class that represents an external worker with a loop and nodeable methods. """ + config_cls: Type[ExternalWorkerConfig] = ExternalWorkerConfig + RUNNING_WORKERS: Dict[str, WeakValueDictionary[str, FuncNodesExternalWorker]] = {} IS_ABSTRACT = True - def __init__(self, workerid) -> None: + def __init__( + self, + workerid, + config: Optional[Union[ExternalWorkerConfig, Dict[str, Any]]] = None, + ) -> None: """ Initializes the FuncNodesExternalWorker class. @@ -24,12 +37,38 @@ def __init__(self, workerid) -> None: delay=1, ) self.uuid = workerid + + self._config = self.config_cls() + try: + self.update_config(config) + except Exception: + pass if self.NODECLASSID not in FuncNodesExternalWorker.RUNNING_WORKERS: FuncNodesExternalWorker.RUNNING_WORKERS[self.NODECLASSID] = ( WeakValueDictionary() ) FuncNodesExternalWorker.RUNNING_WORKERS[self.NODECLASSID][self.uuid] = self + def update_config( + self, config: Optional[Union[ExternalWorkerConfig, Dict[str, Any]]] = None + ): + if config is None: + return + preconfig = config if isinstance(config, dict) else config.model_dump() + self._config = self.config_cls(**{**self._config.model_dump(), **preconfig}) + self.post_config_update() + return self._config + + def post_config_update(self): + """ + This method is called after the config is updated to allow the worker to perform any necessary actions. + """ + pass + + @property + def config(self) -> ExternalWorkerConfig: + return self._config + @classmethod def running_instances(cls) -> List[FuncNodesExternalWorker]: """ @@ -67,6 +106,7 @@ def serialize(self) -> FuncNodesExternalWorkerJson: nodeclassid=self.NODECLASSID, running=self.running, name=self.name, + config=self.config.model_dump(mode="json"), ) diff --git a/src/funcnodes_worker/websocket.py b/src/funcnodes_worker/websocket.py index 2dafd18..7af65fe 100644 --- a/src/funcnodes_worker/websocket.py +++ b/src/funcnodes_worker/websocket.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import List, Optional, Tuple, Dict, Union -from aiohttp import web, WSCloseCode +from aiohttp import web, WSCloseCode, client_exceptions from pathlib import Path try: @@ -64,7 +64,7 @@ async def process_queue(self): await asyncio.wait_for(self.ws.send_bytes(msg), timeout=2) else: await asyncio.wait_for(self.ws.send_str(msg), timeout=2) - except web.ClientError: + except client_exceptions.ClientError: pass except Exception as exc: self.logger.exception("Error sending message", exc_info=exc) diff --git a/src/funcnodes_worker/worker.py b/src/funcnodes_worker/worker.py index 2c6686c..e5a6c22 100644 --- a/src/funcnodes_worker/worker.py +++ b/src/funcnodes_worker/worker.py @@ -289,10 +289,24 @@ def start_local_worker( [EXTERNALWORKERLIB, worker_instance.uuid], ) + def _inner_update_worker_shelf(*args, **kwargs): + self._update_worker_shelf(worker_instance) + + worker_instance.on("nodes_update", _inner_update_worker_shelf) + self._client.request_save() return worker_instance + def _update_worker_shelf(self, worker_instance: FuncNodesExternalWorker): + self._client.nodespace.lib.remove_shelf_path( + [EXTERNALWORKERLIB, worker_instance.uuid] + ) + self._client.nodespace.lib.add_nodes( + worker_instance.get_all_nodeclasses(), + [EXTERNALWORKERLIB, worker_instance.uuid], + ) + def start_local_worker_by_id(self, worker_id: str): for worker_class in self.worker_classes: if worker_class.NODECLASSID == worker_id: @@ -370,6 +384,12 @@ async def stop_local_workers_by_id(self, worker_id: str) -> bool: return True return False + async def get_local_worker_by_id(self, class_id: str, worker_id: str): + if class_id in FuncNodesExternalWorker.RUNNING_WORKERS: + if worker_id in FuncNodesExternalWorker.RUNNING_WORKERS[class_id]: + return FuncNodesExternalWorker.RUNNING_WORKERS[class_id][worker_id] + return None + class SaveLoop(CustomLoop): def __init__(self, client: Worker, delay=5) -> None: @@ -641,6 +661,8 @@ def _check_process_file(self, hard: bool = False): self.loop_manager.async_call(self.run_cmd(cmd)) else: if psutil.pid_exists(cmd) and cmd != os.getpid(): + if self._runstate != "stopped": + self.stop(save=False) raise RuntimeError("Worker already running") except RuntimeError as e: raise e @@ -1035,6 +1057,7 @@ def update_external_worker( worker_id: str, class_id: str, name: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, ): worker_instance = FuncNodesExternalWorker.RUNNING_WORKERS.get(class_id, {}).get( worker_id @@ -1043,6 +1066,8 @@ def update_external_worker( raise ValueError(f"Worker {worker_id} not found") if name is not None: worker_instance.name = name + if config is not None: + worker_instance.update_config(config) self.loop_manager.async_call(self.worker_event("external_worker_update")) @@ -1054,6 +1079,21 @@ async def remove_external_worker(self, worker_id: str, class_id: str): return res + @exposed_method() + async def get_external_worker_config( + self, worker_id: str, class_id: str + ) -> Dict[str, Dict[str, Any]]: + worker_instance = await self.local_worker_lookup_loop.get_local_worker_by_id( + class_id, worker_id + ) + if worker_instance is None: + raise ValueError(f"Worker {worker_id} ({class_id}) not found") + return { + "jsonSchema": worker_instance.config.model_json_schema(), + "uiSchema": None, + "formData": worker_instance.config.model_dump(mode="json"), + } + # endregion local worker # region states @exposed_method() @@ -1293,6 +1333,8 @@ async def load(self, data: WorkerState | str | None = None): w = self.add_local_worker(worker, instance["uuid"]) if "name" in instance: w.name = instance["name"] + if "config" in instance: + w.update_config(instance["config"]) found = True if not found: self.logger.warning(f"External worker {worker_id} not found") @@ -2094,11 +2136,12 @@ def init_and_run_forever( worker.run_forever() worker.logger.debug("Worker initialized and running stopped") - def stop(self): + def stop(self, save: bool = True): if self.is_running(): self.loop_manager.async_call(self.worker_event("stopping")) self.runstate = "stopped" - self.save() + if save: + self.save() self._save_disabled = True self.loop_manager.stop() @@ -2117,7 +2160,10 @@ def is_running(self): return self.loop_manager.running def cleanup(self): - self.runstate = "removed" + try: + self.runstate = "removed" + except NameError: + pass if self.is_running(): # pragma: no cover self.stop() self.loop_manager.stop() From a06f00b37f060417804d191af3ac89dc15144a38 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Wed, 12 Nov 2025 23:15:49 +0100 Subject: [PATCH 02/19] feat(worker): enhance FuncNodesExternalWorker with nodeshelf property and logging for configuration updates --- src/funcnodes_worker/external_worker.py | 29 ++++++++++++++++++++++++- src/funcnodes_worker/worker.py | 20 +++++++++++++++-- tests/test_external_worker.py | 3 ++- tests/test_worker.py | 1 - 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/funcnodes_worker/external_worker.py b/src/funcnodes_worker/external_worker.py index b8051c7..43eb8ea 100644 --- a/src/funcnodes_worker/external_worker.py +++ b/src/funcnodes_worker/external_worker.py @@ -1,9 +1,17 @@ from __future__ import annotations from typing import Dict, List, TypedDict, Union, Any, Optional, Type from funcnodes_worker.loop import CustomLoop -from funcnodes_core import NodeClassMixin, JSONEncoder, Encdata, EventEmitterMixin +from funcnodes_core import ( + NodeClassMixin, + JSONEncoder, + Encdata, + EventEmitterMixin, + Shelf, + FUNCNODES_LOGGER, +) from weakref import WeakValueDictionary from pydantic import BaseModel +from weakref import ref class ExternalWorkerConfig(BaseModel): @@ -57,6 +65,7 @@ def update_config( preconfig = config if isinstance(config, dict) else config.model_dump() self._config = self.config_cls(**{**self._config.model_dump(), **preconfig}) self.post_config_update() + FUNCNODES_LOGGER.info(f"config updated for worker {self.uuid}: {self._config}") return self._config def post_config_update(self): @@ -69,6 +78,24 @@ def post_config_update(self): def config(self) -> ExternalWorkerConfig: return self._config + @property + def nodeshelf(self) -> Optional[ref[Shelf]]: + ns = self.get_nodeshelf() + print(f"nodeshelf: {ns}") + if ns is None: + return None + if ns.name != self.uuid: + ns = Shelf( + name=self.uuid, + description=ns.description, + nodes=list(ns.nodes), + subshelves=list(ns.subshelves), + ) + return ref(ns) + + def get_nodeshelf(self) -> Optional[Shelf]: + return None + @classmethod def running_instances(cls) -> List[FuncNodesExternalWorker]: """ diff --git a/src/funcnodes_worker/worker.py b/src/funcnodes_worker/worker.py index e5a6c22..2f4a1ca 100644 --- a/src/funcnodes_worker/worker.py +++ b/src/funcnodes_worker/worker.py @@ -106,7 +106,6 @@ class WorkerState(TypedDict): backend: NodeSpaceJSON view: ViewState meta: MetaInfo - dependencies: dict[str, List[str]] external_workers: Dict[str, List[FuncNodesExternalWorkerJson]] @@ -276,6 +275,10 @@ def start_local_worker( ): if worker_class not in self.worker_classes: self.worker_classes.append(worker_class) + + self._client.logger.info( + f"starting local worker {worker_class.NODECLASSID} {worker_id}" + ) worker_instance: FuncNodesExternalWorker = worker_class(workerid=worker_id) worker_instance.on( @@ -288,6 +291,11 @@ def start_local_worker( worker_instance.get_all_nodeclasses(), [EXTERNALWORKERLIB, worker_instance.uuid], ) + worker_nodeshelf = worker_instance.nodeshelf + if worker_nodeshelf is not None: + self._client.nodespace.lib.add_external_shelf( + worker_instance.nodeshelf, [EXTERNALWORKERLIB] + ) def _inner_update_worker_shelf(*args, **kwargs): self._update_worker_shelf(worker_instance) @@ -302,10 +310,19 @@ def _update_worker_shelf(self, worker_instance: FuncNodesExternalWorker): self._client.nodespace.lib.remove_shelf_path( [EXTERNALWORKERLIB, worker_instance.uuid] ) + try: + self._client.nodespace.lib.remove_shelf_path([worker_instance.uuid]) + except ValueError: + pass self._client.nodespace.lib.add_nodes( worker_instance.get_all_nodeclasses(), [EXTERNALWORKERLIB, worker_instance.uuid], ) + worker_nodeshelf = worker_instance.nodeshelf + if worker_nodeshelf is not None: + self._client.nodespace.lib.add_external_shelf( + worker_nodeshelf, [EXTERNALWORKERLIB] + ) def start_local_worker_by_id(self, worker_id: str): for worker_class in self.worker_classes: @@ -1167,7 +1184,6 @@ def get_save_state(self) -> WorkerState: "backend": saving.serialize_nodespace_for_saving(self.nodespace), "view": ws, "meta": self.get_meta(), - "dependencies": self.nodespace.lib.get_dependencies(), "external_workers": { workerclass.NODECLASSID: [ w_instance.serialize() diff --git a/tests/test_external_worker.py b/tests/test_external_worker.py index 8559ae2..79cfb6b 100644 --- a/tests/test_external_worker.py +++ b/tests/test_external_worker.py @@ -138,6 +138,7 @@ def test(self, a: int) -> int: "nodeclassid": "testexternalworker", "running": False, "uuid": "test", + "config": {}, }, ) @@ -184,7 +185,7 @@ async def asyncSetUp(self) -> None: self._loop = asyncio.get_event_loop() self.runtask = self._loop.create_task(self.retmoteworker.run_forever_async()) t = time.time() - while not self.retmoteworker.loop_manager.running and time.time() - t < 10: + while not self.retmoteworker.loop_manager.running and time.time() - t < 50: if self.runtask.done(): if self.runtask.exception(): raise self.runtask.exception() diff --git a/tests/test_worker.py b/tests/test_worker.py index a685461..a3c6491 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -415,7 +415,6 @@ async def test_load(self): }, view={}, meta={}, - dependencies={}, external_workers={}, ) From fb6dddf5b4ad8aae2655d94407ecd6bf372bcd59 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Wed, 12 Nov 2025 23:42:47 +0100 Subject: [PATCH 03/19] fix(tests): correct asyncio_default_fixture_loop_scope format in pytest configuration --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3c9bbd8..7e5d9a0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] asyncio_mode=auto -asyncio_default_fixture_loop_scope="function" +asyncio_default_fixture_loop_scope=function addopts = -p no:logging From ced9c56e17866d0923649078639e27e671c05632 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Wed, 12 Nov 2025 23:43:14 +0100 Subject: [PATCH 04/19] feat(websocket): implement graceful client connection closure and enhance message enqueue handling --- src/funcnodes_worker/websocket.py | 62 +++++++++++++++++++++++-------- tests/test_client_connection.py | 48 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 tests/test_client_connection.py diff --git a/src/funcnodes_worker/websocket.py b/src/funcnodes_worker/websocket.py index 7af65fe..27118b3 100644 --- a/src/funcnodes_worker/websocket.py +++ b/src/funcnodes_worker/websocket.py @@ -2,6 +2,7 @@ from typing import List, Optional, Tuple, Dict, Union from aiohttp import web, WSCloseCode, client_exceptions from pathlib import Path +import contextlib try: import aiohttp_cors @@ -54,24 +55,50 @@ def __init__(self, ws: web.WebSocketResponse, logger): self.queue = asyncio.Queue(maxsize=1000) # Adjust maxsize as needed self.logger = logger self.send_task = asyncio.create_task(self.process_queue()) + self._close_lock = asyncio.Lock() + self._closed = False async def process_queue(self): - while True: - msg: Union[str, bytes] = await self.queue.get() - try: - # Apply a timeout to avoid waiting indefinitely for a slow client - if isinstance(msg, bytes): - await asyncio.wait_for(self.ws.send_bytes(msg), timeout=2) + try: + while True: + msg: Union[str, bytes] = await self.queue.get() + try: + # Apply a timeout to avoid waiting indefinitely for a slow client + if isinstance(msg, bytes): + await asyncio.wait_for(self.ws.send_bytes(msg), timeout=2) + else: + await asyncio.wait_for(self.ws.send_str(msg), timeout=2) + except client_exceptions.ClientError: + pass + except Exception as exc: + self.logger.exception("Error sending message", exc_info=exc) + finally: + self.queue.task_done() + except asyncio.CancelledError: + # Drain pending messages to keep task_done counts balanced. + while not self.queue.empty(): + try: + self.queue.get_nowait() + except asyncio.QueueEmpty: + break else: - await asyncio.wait_for(self.ws.send_str(msg), timeout=2) - except client_exceptions.ClientError: - pass - except Exception as exc: - self.logger.exception("Error sending message", exc_info=exc) - finally: - self.queue.task_done() + self.queue.task_done() + raise - async def enqueue(self, msg: str): + async def close(self): + async with self._close_lock: + if self._closed: + return + self._closed = True + if not self.send_task.done(): + self.send_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self.send_task + + async def enqueue(self, msg: Union[str, bytes]): + if self._closed: + self.logger.debug("Dropping message for closed client connection") + return try: self.queue.put_nowait(msg) except asyncio.QueueFull: @@ -163,7 +190,9 @@ async def _handle_connection(self, request: web.Request): FUNCNODES_LOGGER.exception(e) finally: self._worker.logger.debug("Client disconnected") - self.clients.remove(client_connection) + await client_connection.close() + if client_connection in self.clients: + self.clients.remove(client_connection) return websocket @@ -340,10 +369,11 @@ async def stop(self): """ # close all clients - for client in self.clients: + for client in list(self.clients): await client.ws.close( code=WSCloseCode.GOING_AWAY, message="Server shutting down" ) + await client.close() self.message_store.clear() diff --git a/tests/test_client_connection.py b/tests/test_client_connection.py new file mode 100644 index 0000000..bfd9ab6 --- /dev/null +++ b/tests/test_client_connection.py @@ -0,0 +1,48 @@ +import asyncio +import logging +from unittest import IsolatedAsyncioTestCase + +from funcnodes_worker.websocket import ClientConnection + + +class DummyWebSocket: + def __init__(self, delay: float = 0.0): + self.delay = delay + self.sent = [] + self.closed = False + + async def send_str(self, msg: str): + await asyncio.sleep(self.delay) + self.sent.append(("str", msg)) + + async def send_bytes(self, data: bytes): + await asyncio.sleep(self.delay) + self.sent.append(("bytes", data)) + + async def close(self, *_, **__): + self.closed = True + + +class TestClientConnection(IsolatedAsyncioTestCase): + async def test_close_cancels_send_task(self): + ws = DummyWebSocket(delay=0.1) + client = ClientConnection(ws, logging.getLogger("test_close")) + + # Enqueue data so the send loop is actively processing. + await client.enqueue("ping") + await asyncio.sleep(0.01) + + await client.close() + + self.assertTrue(client.send_task.done()) + self.assertTrue(client.queue.empty()) + + async def test_enqueue_after_close_is_noop(self): + ws = DummyWebSocket() + client = ClientConnection(ws, logging.getLogger("test_enqueue")) + + await client.close() + await client.enqueue("ignored") + + self.assertTrue(client.send_task.done()) + self.assertFalse(ws.sent) From 26927c03738b55bfa6e24f6e4ab31c9be2954b91 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Thu, 27 Nov 2025 19:15:40 +0100 Subject: [PATCH 05/19] fix(loop): handle closed or missing event loop for tasks Fallback to a running loop when available, warn and skip when none is usable, and suppress deprecation warnings around event loop lookup. --- src/funcnodes_worker/loop.py | 52 +++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/src/funcnodes_worker/loop.py b/src/funcnodes_worker/loop.py index c69d669..1150cae 100644 --- a/src/funcnodes_worker/loop.py +++ b/src/funcnodes_worker/loop.py @@ -3,6 +3,7 @@ import asyncio from typing import List, Optional import logging +import warnings from funcnodes_core import NodeSpace import time import weakref @@ -108,13 +109,25 @@ def __init__(self, worker) -> None: def reset_loop(self): try: - self._loop = asyncio.get_event_loop() - except RuntimeError as e: - if str(e).startswith("There is no current event loop in thread"): - self._loop = asyncio.new_event_loop() - asyncio.set_event_loop(self._loop) - else: - raise + # Try to get the running loop first (Python 3.7+) + self._loop = asyncio.get_running_loop() + except RuntimeError: + # No running loop, try to get the current event loop + # Suppress deprecation warning for get_event_loop() in Python 3.13+ + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + try: + self._loop = asyncio.get_event_loop() + except RuntimeError as e: + error_msg = str(e) + if ( + "There is no current event loop" in error_msg + or "There is no running event loop" in error_msg + ): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + else: + raise def add_loop(self, loop: CustomLoop): if self._running: @@ -157,6 +170,31 @@ def remove_loop(self, loop: CustomLoop): task.cancel() def async_call(self, croutine: asyncio.Coroutine): + # Check if the loop is closed or not running + if self._loop.is_closed(): + # Try to get the running loop instead + try: + running_loop = asyncio.get_running_loop() + return running_loop.create_task(croutine) + except RuntimeError: + # No running loop available, skip creating the task + worker = self._worker() + if worker is not None: + worker.logger.warning( + "Cannot create task: event loop is closed and no running loop available" + ) + return None + + # Check if the loop is running + if not self._loop.is_running(): + # Try to get the running loop instead + try: + running_loop = asyncio.get_running_loop() + return running_loop.create_task(croutine) + except RuntimeError: + # No running loop available, but our loop exists, try to use it + pass + return self._loop.create_task(croutine) def __del__(self): From bde7dccfd01d9104663948198560993d714705d3 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Thu, 27 Nov 2025 19:16:25 +0100 Subject: [PATCH 06/19] chore(remote-worker): standardize exception logging variable --- src/funcnodes_worker/remote_worker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/funcnodes_worker/remote_worker.py b/src/funcnodes_worker/remote_worker.py index 71fbbba..eb4eca0 100644 --- a/src/funcnodes_worker/remote_worker.py +++ b/src/funcnodes_worker/remote_worker.py @@ -143,13 +143,13 @@ async def receive_message(self, json_msg: dict, **sendkwargs): await self._handle_cmd_msg(json_msg, json_response=True, **sendkwargs) if json_msg["type"] == "ping": await self.send('{"type": "pong"}') - except Exception as e: - self.logger.exception(e) + except Exception as exc: + self.logger.exception(exc) await self.send( ErrorMessage( type="error", - error=str(e), - tb=traceback.format_exception(e), + error=str(exc), + tb=traceback.format_exception(exc), id=json_msg.get("id"), ) ) From 26d0057c6dcbbbca94b4e31e69ffcb6afc44280a Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Thu, 27 Nov 2025 19:17:45 +0100 Subject: [PATCH 07/19] feat(external-worker): support exportable configs and nodeshelf updates Allow external worker configs to omit non-export fields during export. Track data paths, validate nodeshelf assignment, and emit updates on set. Provide optional export flag in serialize for packaging worker state. --- pyproject.toml | 4 +- src/funcnodes_worker/external_worker.py | 87 +++++++++--- uv.lock | 172 ++++++++++++++++++++++-- 3 files changed, 237 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a06ef35..1403ba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,9 +8,10 @@ authors = [{name = "Julian Kimmig", email = "julian.kimmig@linkdlab.de"}] requires-python = ">=3.11" dependencies = [ "asynctoolkit>=0.1.1", - "funcnodes-core>=1.0.1", + "funcnodes-core>=2.0.0a3", "packaging>=24.2", "pip>=25.0.1", + "pydantic>=2.12.4", "python-slugify>=8.0.4", ] @@ -25,6 +26,7 @@ dev = [ "funcnodes-worker[all]", "funcnodes>=1.0.0", "objgraph>=3.6.2", + "pytest-funcnodes>=0.2.0", ] [project.optional-dependencies] diff --git a/src/funcnodes_worker/external_worker.py b/src/funcnodes_worker/external_worker.py index 43eb8ea..128d1f6 100644 --- a/src/funcnodes_worker/external_worker.py +++ b/src/funcnodes_worker/external_worker.py @@ -1,5 +1,6 @@ from __future__ import annotations -from typing import Dict, List, TypedDict, Union, Any, Optional, Type +from pathlib import Path +from typing import Dict, List, TypedDict, Union, Any, Optional, Type, ClassVar from funcnodes_worker.loop import CustomLoop from funcnodes_core import ( NodeClassMixin, @@ -19,6 +20,27 @@ class ExternalWorkerConfig(BaseModel): A class that represents the configuration of an external worker. """ + EXPORT_EXCLUDE_FIELDS: ClassVar[set[str]] = set() + + @classmethod + def export_exclude_fields(cls) -> set[str]: + """Returns field names that should be removed when exporting config.""" + excluded = set(getattr(cls, "EXPORT_EXCLUDE_FIELDS", set())) + fields = getattr(cls, "model_fields", None) or getattr(cls, "__fields__", {}) + for name, field in fields.items(): + extra = getattr(field, "json_schema_extra", None) + if extra is None and hasattr(field, "field_info"): + extra = getattr(field.field_info, "extra", {}) or getattr( + field.field_info, "json_schema_extra", None + ) + if extra and extra.get("export") is False: + excluded.add(name) + return excluded + + def exportable_dict(self) -> dict: + """Serialize config without export-excluded fields.""" + return self.model_dump(mode="json", exclude=self.export_exclude_fields()) + class FuncNodesExternalWorker(NodeClassMixin, EventEmitterMixin, CustomLoop): """ @@ -34,6 +56,7 @@ def __init__( self, workerid, config: Optional[Union[ExternalWorkerConfig, Dict[str, Any]]] = None, + data_path: Optional[str] = None, ) -> None: """ Initializes the FuncNodesExternalWorker class. @@ -45,8 +68,9 @@ def __init__( delay=1, ) self.uuid = workerid - + self._nodeshelf: Optional[Shelf] = None self._config = self.config_cls() + self._data_path: Optional[Path] = Path(data_path) if data_path else None try: self.update_config(config) except Exception: @@ -57,6 +81,23 @@ def __init__( ) FuncNodesExternalWorker.RUNNING_WORKERS[self.NODECLASSID][self.uuid] = self + @property + def data_path(self) -> Optional[Path]: + if self._data_path is None: + return None + if not self._data_path.exists(): + self._data_path.mkdir(parents=True, exist_ok=True) + return self._data_path + + @data_path.setter + def data_path(self, data_path: Optional[Path]): + if data_path is None: + self._data_path = None + else: + self._data_path = data_path.resolve() + if not self._data_path.exists(): + self._data_path.mkdir(parents=True, exist_ok=True) + def update_config( self, config: Optional[Union[ExternalWorkerConfig, Dict[str, Any]]] = None ): @@ -64,7 +105,10 @@ def update_config( return preconfig = config if isinstance(config, dict) else config.model_dump() self._config = self.config_cls(**{**self._config.model_dump(), **preconfig}) - self.post_config_update() + try: + self.post_config_update() + except Exception as e: + FUNCNODES_LOGGER.exception(e) FUNCNODES_LOGGER.info(f"config updated for worker {self.uuid}: {self._config}") return self._config @@ -81,20 +125,24 @@ def config(self) -> ExternalWorkerConfig: @property def nodeshelf(self) -> Optional[ref[Shelf]]: ns = self.get_nodeshelf() - print(f"nodeshelf: {ns}") if ns is None: return None - if ns.name != self.uuid: - ns = Shelf( - name=self.uuid, - description=ns.description, - nodes=list(ns.nodes), - subshelves=list(ns.subshelves), - ) - return ref(ns) + return ref(ns) # + + @nodeshelf.setter + def nodeshelf(self, ns: Optional[Shelf]): + self.set_nodeshelf(ns) def get_nodeshelf(self) -> Optional[Shelf]: - return None + return self._nodeshelf + + def set_nodeshelf(self, ns: Optional[Shelf]): + if ns is None: + self._nodeshelf = ns + if not isinstance(ns, Shelf): + raise ValueError("ns must be a Shelf or None") + self._nodeshelf = ns + self.emit("nodes_update") @classmethod def running_instances(cls) -> List[FuncNodesExternalWorker]: @@ -124,18 +172,26 @@ async def stop(self): self.cleanup() await super().stop() - def serialize(self) -> FuncNodesExternalWorkerJson: + def serialize(self, export: bool = False) -> FuncNodesExternalWorkerJson: """ Serializes the FuncNodesExternalWorker class. """ + cfg = ( + self.config.exportable_dict() + if export and hasattr(self.config, "exportable_dict") + else self.config.model_dump(mode="json") + ) return FuncNodesExternalWorkerJson( uuid=self.uuid, nodeclassid=self.NODECLASSID, running=self.running, name=self.name, - config=self.config.model_dump(mode="json"), + config=cfg, ) + async def loop(self): + pass + class FuncNodesExternalWorkerJson(TypedDict): """ @@ -146,6 +202,7 @@ class FuncNodesExternalWorkerJson(TypedDict): nodeclassid: str running: bool name: str + config: dict def encode_external_worker(obj, preview=False): # noqa: F841 diff --git a/uv.lock b/uv.lock index bf9880c..1be2deb 100644 --- a/uv.lock +++ b/uv.lock @@ -104,6 +104,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597, upload-time = "2024-12-13T17:10:38.469Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "appdirs" version = "1.4.4" @@ -297,15 +306,15 @@ wheels = [ [[package]] name = "exposedfunctionality" -version = "0.3.20" +version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/14/2d/293b8dd4a8d021736e3f8c2fe11d44050998886da0cab3f4b8c90b746443/exposedfunctionality-0.3.20.tar.gz", hash = "sha256:f45bfc747177a9306c2c53893818910b32e264246a961ad605202d37bcdbf674", size = 23887, upload-time = "2025-02-24T12:23:22.874Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/e2/b46eec09792009ccee3fa3d8fba3540be1478655612fae8ba3f6f1b871ca/exposedfunctionality-1.0.2.tar.gz", hash = "sha256:6d4a55223041ef91d955d8016c2afbbcedb8e70fd1fdec76dfe69073d9da6f79", size = 68056, upload-time = "2025-09-03T09:38:58.119Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/f9/4b440f7a6592d4f94e4eb084aff7ec4025e8f86292ab4633d6499766252c/exposedfunctionality-0.3.20-py3-none-any.whl", hash = "sha256:aa009389ebf7230493f2b960acd2d5142145eae9fc8f2f679bca9eaa8d1e53c1", size = 27090, upload-time = "2025-02-24T12:23:21.374Z" }, + { url = "https://files.pythonhosted.org/packages/5f/42/111fdd98410f4cc3c2bcb8cf1e92132f214dc5872b52db5ec84ccf7d16ba/exposedfunctionality-1.0.2-py3-none-any.whl", hash = "sha256:6a58bbd700044a80b11b4fbeaf4458fd034f455847be9036718d9238574d0bc5", size = 28831, upload-time = "2025-09-03T09:38:57.051Z" }, ] [[package]] @@ -420,18 +429,19 @@ wheels = [ [[package]] name = "funcnodes-core" -version = "1.0.1" +version = "2.0.0a3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, { name = "exposedfunctionality" }, + { name = "pytest-funcnodes" }, { name = "python-dotenv" }, { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/95/bd171144469aa9ac13eaa607b1f721c26ddf762e747f408ea4a8ff0f86c6/funcnodes_core-1.0.1.tar.gz", hash = "sha256:df3f4f351e71e34d82c189bd2a2597d6ce21e05bea987f01b17582b546d4416b", size = 176410, upload-time = "2025-09-01T12:46:06.231Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/02/683c1e20132b9d865b747ada8829c279c759b15ac25d8e289a797515e61a/funcnodes_core-2.0.0a3.tar.gz", hash = "sha256:1c8250d7cfb81a3b70bf47e552da5ea4e22dbbbcc11c0f439fbc56046aeb6108", size = 189872, upload-time = "2025-11-27T11:11:28.071Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/fc/ee82da057c35348e3da3dae9f7ef6667ff739b792c697a80a6ecc4e88be0/funcnodes_core-1.0.1-py3-none-any.whl", hash = "sha256:de28abad9e497fe91e23f78415992cf78ca6e2bde5baa977ac867852f631cff8", size = 88798, upload-time = "2025-09-01T12:46:05.007Z" }, + { url = "https://files.pythonhosted.org/packages/3c/ca/ca0ba069573e33d4831ab9789c7f90978959a8265666063dd0b69db5fac6/funcnodes_core-2.0.0a3-py3-none-any.whl", hash = "sha256:11ab6c16eba70a973061f291f1eb5ca4b66ecda71ea60d739f5131a4b96bde91", size = 91647, upload-time = "2025-11-27T11:11:26.728Z" }, ] [[package]] @@ -471,6 +481,7 @@ dependencies = [ { name = "funcnodes-core" }, { name = "packaging" }, { name = "pip" }, + { name = "pydantic" }, { name = "python-slugify" }, ] @@ -503,6 +514,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-funcnodes" }, { name = "snakeviz" }, { name = "vulture" }, ] @@ -512,10 +524,11 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'http'" }, { name = "aiohttp-cors", marker = "extra == 'http'" }, { name = "asynctoolkit", specifier = ">=0.1.1" }, - { name = "funcnodes-core", specifier = ">=1.0.1" }, + { name = "funcnodes-core", specifier = ">=2.0.0a3" }, { name = "funcnodes-worker", extras = ["venv", "http", "subprocess-monitor"], marker = "extra == 'all'" }, { name = "packaging", specifier = ">=24.2" }, { name = "pip", specifier = ">=25.0.1" }, + { name = "pydantic", specifier = ">=2.12.4" }, { name = "python-slugify", specifier = ">=8.0.4" }, { name = "requests", marker = "extra == 'http'" }, { name = "subprocess-monitor", marker = "extra == 'subprocess-monitor'", specifier = ">=0.3.0" }, @@ -532,6 +545,7 @@ dev = [ { name = "pre-commit", specifier = ">=4.1.0" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.3" }, + { name = "pytest-funcnodes", specifier = ">=0.2.0" }, { name = "snakeviz", specifier = ">=2.2.2" }, { name = "vulture", specifier = ">=2.14" }, ] @@ -876,6 +890,118 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, ] +[[package]] +name = "pydantic" +version = "2.12.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + [[package]] name = "pygments" version = "2.19.1" @@ -912,6 +1038,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/7f/338843f449ace853647ace35870874f69a764d251872ed1b4de9f234822c/pytest_asyncio-0.26.0-py3-none-any.whl", hash = "sha256:7b51ed894f4fbea1340262bdae5135797ebbe21d8638978e35d31c6d19f72fb0", size = 19694, upload-time = "2025-03-25T06:22:27.807Z" }, ] +[[package]] +name = "pytest-funcnodes" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "funcnodes-core" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/fd/82d7b03f478fee54b9d1b51509191b9ca76b2e6634477c01c4f591a6754a/pytest_funcnodes-1.0.0.tar.gz", hash = "sha256:3e076d91e1708723f750aed0bf53d817a04be62e6645cbda14b06e51b565e0f8", size = 46492, upload-time = "2025-11-27T11:18:28.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/3c/171f60cd0ac290d3625fb04d8823bc73993e55512a6f7428a0908086f118/pytest_funcnodes-1.0.0-py3-none-any.whl", hash = "sha256:c38e875830d18618e93dd1343d7fc6fe5ee85f47144fd05039fce6014b6d6d1b", size = 31817, upload-time = "2025-11-27T11:18:26.656Z" }, +] + [[package]] name = "python-dotenv" version = "1.1.0" @@ -1161,11 +1301,23 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.13.2" +version = "4.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] From a30647b08a5bb7fb348f33f1a0d3af5ee3ee6297 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Thu, 27 Nov 2025 19:19:10 +0100 Subject: [PATCH 08/19] fix(worker): align external worker shelf updates and export Ensure external worker shelves are refreshed and references cleaned when workers restart, passing data paths to instances and forcing export mode when saving worker state. --- src/funcnodes_worker/worker.py | 82 +++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/src/funcnodes_worker/worker.py b/src/funcnodes_worker/worker.py index 2f4a1ca..9fe7834 100644 --- a/src/funcnodes_worker/worker.py +++ b/src/funcnodes_worker/worker.py @@ -1,6 +1,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from logging.handlers import RotatingFileHandler +import gc from functools import wraps from typing import ( List, @@ -199,7 +199,7 @@ def path(self): p = self._path if not p.exists(): - os.mkdir(p) + p.mkdir(parents=True, exist_ok=True) if str(p) not in sys.path: sys.path.insert(0, str(p)) @@ -279,7 +279,10 @@ def start_local_worker( self._client.logger.info( f"starting local worker {worker_class.NODECLASSID} {worker_id}" ) - worker_instance: FuncNodesExternalWorker = worker_class(workerid=worker_id) + worker_instance: FuncNodesExternalWorker = worker_class( + workerid=worker_id, + data_path=self._client.data_path / "external_workers" / worker_id, + ) worker_instance.on( "stopping", @@ -287,41 +290,47 @@ def start_local_worker( ) self._client.loop_manager.add_loop(worker_instance) - self._client.nodespace.lib.add_nodes( - worker_instance.get_all_nodeclasses(), - [EXTERNALWORKERLIB, worker_instance.uuid], - ) - worker_nodeshelf = worker_instance.nodeshelf - if worker_nodeshelf is not None: - self._client.nodespace.lib.add_external_shelf( - worker_instance.nodeshelf, [EXTERNALWORKERLIB] - ) - def _inner_update_worker_shelf(*args, **kwargs): self._update_worker_shelf(worker_instance) worker_instance.on("nodes_update", _inner_update_worker_shelf) + self._update_worker_shelf(worker_instance) + self._client.request_save() return worker_instance def _update_worker_shelf(self, worker_instance: FuncNodesExternalWorker): - self._client.nodespace.lib.remove_shelf_path( - [EXTERNALWORKERLIB, worker_instance.uuid] - ) + shelf_path = [EXTERNALWORKERLIB, worker_instance.uuid] try: - self._client.nodespace.lib.remove_shelf_path([worker_instance.uuid]) + self._client.nodespace.lib.remove_shelf_path(shelf_path) except ValueError: pass + + worker_nodeshelf_ref = worker_instance.nodeshelf + worker_nodeshelf_obj = ( + worker_nodeshelf_ref() if worker_nodeshelf_ref is not None else None + ) + if worker_nodeshelf_obj is not None: + try: + self._client.nodespace.lib.remove_shelf_path( + [*shelf_path, worker_nodeshelf_obj.name] + ) + except ValueError: + pass + # perform a gc collect to remove any references to the old nodeshelf + gc.collect() + self._client.nodespace.lib.add_nodes( worker_instance.get_all_nodeclasses(), - [EXTERNALWORKERLIB, worker_instance.uuid], + shelf_path, ) - worker_nodeshelf = worker_instance.nodeshelf - if worker_nodeshelf is not None: - self._client.nodespace.lib.add_external_shelf( - worker_nodeshelf, [EXTERNALWORKERLIB] + # Reuse worker_nodeshelf_obj instead of accessing the property again + # to avoid calling get_nodeshelf() twice + if worker_nodeshelf_obj is not None: + self._client.nodespace.lib.add_subshelf_weak( + worker_nodeshelf_ref, shelf_path ) def start_local_worker_by_id(self, worker_id: str): @@ -623,13 +632,13 @@ def __init__( self.logger.setLevel("DEBUG") self.logger.info("Init Worker %s", self.__class__.__name__) - self.logger.addHandler( - RotatingFileHandler( - self.data_path / "worker.log", - maxBytes=100000, - backupCount=5, - ) - ) + # self.logger.addHandler( + # RotatingFileHandler( + # self.data_path / "worker.log", + # maxBytes=100000, + # backupCount=5, + # ) + # ) self._exposed_methods = get_exposed_methods(self) self._progress_state: ProgressState = { @@ -929,9 +938,9 @@ def export_worker(self, with_files: bool = True) -> bytes: ) zip_file.writestr( "state", - json.dumps(self.get_save_state(), cls=JSONEncoder, indent=2).encode( - "utf-8" - ), + json.dumps( + self.get_save_state(export=True), cls=JSONEncoder, indent=2 + ).encode("utf-8"), ) if self.venvmanager: tomlpath = self.data_path / "pyproject.toml" @@ -1083,9 +1092,12 @@ def update_external_worker( raise ValueError(f"Worker {worker_id} not found") if name is not None: worker_instance.name = name + if config is not None: worker_instance.update_config(config) - + # Note: _update_worker_shelf will be called automatically via the + # "nodes_update" event handler registered in start_local_worker, + # so we don't need to call it directly here. self.loop_manager.async_call(self.worker_event("external_worker_update")) @exposed_method() @@ -1177,7 +1189,7 @@ def upload(self, data: Union[bytes, str], filename: Path) -> Path: return filename @exposed_method() - def get_save_state(self) -> WorkerState: + def get_save_state(self, export: bool = False) -> WorkerState: ws = self.view_state() ws.pop("nodes", None) data: WorkerState = { @@ -1186,7 +1198,7 @@ def get_save_state(self) -> WorkerState: "meta": self.get_meta(), "external_workers": { workerclass.NODECLASSID: [ - w_instance.serialize() + w_instance.serialize(export=export) for w_instance in workerclass.running_instances() ] for workerclass in self.local_worker_lookup_loop.worker_classes From 9413bc5d53d7c30c08970265f5f6ef50d244ac49 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Thu, 27 Nov 2025 19:30:10 +0100 Subject: [PATCH 09/19] test(worker): migrate fixtures and cover external worker exports Switch tests to pytest fixtures with pytest-funcnodes setup/teardown. Add coverage for nodeshelf refreshes and export filtering of configs. --- tests/test_external_worker.py | 134 +++- tests/test_loops.py | 17 +- tests/test_socketworker.py | 10 +- tests/test_worker.py | 1132 +++++++++++++++++++-------------- tests/test_ws_worker.py | 18 +- 5 files changed, 793 insertions(+), 518 deletions(-) diff --git a/tests/test_external_worker.py b/tests/test_external_worker.py index 79cfb6b..9174c3b 100644 --- a/tests/test_external_worker.py +++ b/tests/test_external_worker.py @@ -1,13 +1,10 @@ +from typing import Optional from unittest import IsolatedAsyncioTestCase +from weakref import ref import funcnodes_core as fn -from funcnodes_core.testing import ( - teardown as fn_teardown, - set_in_test as fn_set_in_test, -) - -fn_set_in_test() +from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown from funcnodes_worker import ( # noqa: E402 FuncNodesExternalWorker, @@ -20,6 +17,7 @@ instance_nodefunction, flatten_shelf, ) +from funcnodes_core.node import register_node # noqa: E402 from funcnodes_worker import CustomLoop # noqa: E402 import time # noqa: E402 import asyncio # noqa: E402 @@ -34,12 +32,6 @@ except ImportError: objgraph = None -fn.FUNCNODES_LOGGER.setLevel(logging.DEBUG) - - -class ExternalWorker_Test(FuncNodesExternalWorker): - pass - class RaiseErrorLogger(logging.Logger): def exception(self, e: Exception): @@ -72,12 +64,12 @@ async def send_bytes(self, *args, **kwargs): class TestExternalWorker(IsolatedAsyncioTestCase): - def test_external_worker_missing_loop(self): - class ExternalWorker1(FuncNodesExternalWorker): - pass + def setUp(self) -> None: + fn_setup() + register_node(workertestnode) - with self.assertRaises(TypeError): - ExternalWorker1() + def tearDown(self) -> None: + fn_teardown() def test_external_worker_missing_nodeclassid(self): with self.assertRaises(ValueError): @@ -199,6 +191,9 @@ async def asyncTearDown(self): async with asyncio.timeout(5): await self.runtask + def setUp(self) -> None: + fn_setup() + def tearDown(self) -> None: if not self.runtask.done(): self.runtask.cancel() @@ -382,3 +377,108 @@ def check_nodes_length(target=0): await asyncio.sleep(0.5) t = time.time() self.assertLessEqual(t - self.retmoteworker.timerloop.last_run, 0.3) + + +@fn.NodeDecorator(node_id="workertestnode") +async def workertestnode(a: int) -> int: + return a + 1 + + +class ExternalWorkerWithNodeShelves(FuncNodesExternalWorker): + NODECLASSID = "testexternalworker_ExternalWorkerWithNodeShelves" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._nodeshelf = fn.Shelf( + name="test", + description="test", + nodes=[ + workertestnode, + ], + ) + + def get_nodeshelf(self) -> Optional[fn.Shelf]: + return self._nodeshelf + + +class TestExternalWorkerWithNodeShelves(IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory(prefix="funcnodes") + self.retmoteworker = _TestWorker(data_path=self.tempdir.name) + self._loop = asyncio.get_event_loop() + self.runtask = self._loop.create_task(self.retmoteworker.run_forever_async()) + t = time.time() + while not self.retmoteworker.loop_manager.running and time.time() - t < 50: + if self.runtask.done(): + if self.runtask.exception(): + raise self.runtask.exception() + await asyncio.sleep(1) + if not self.retmoteworker.loop_manager.running: + raise Exception("Worker not running") + + async def asyncTearDown(self): + self.retmoteworker.stop() + + async with asyncio.timeout(5): + await self.runtask + + def setUp(self) -> None: + fn_setup() + # register_node(workertestnode) + + def tearDown(self) -> None: + if not self.runtask.done(): + self.runtask.cancel() + + fn_teardown() + self.tempdir.cleanup() + return super().tearDown() + + async def test_external_worker_nodes(self): + worker = self.retmoteworker.add_local_worker( + ExternalWorkerWithNodeShelves, "test_external_worker_nodes" + ) + + assert isinstance(worker, ExternalWorkerWithNodeShelves) + + assert worker.get_nodeshelf() is not None + assert worker.get_nodeshelf().name == "test" + assert worker.get_nodeshelf().nodes == [workertestnode] + + assert isinstance(worker.nodeshelf, ref) + assert worker.nodeshelf() is not None + assert worker.nodeshelf().name == "test" + assert worker.nodeshelf().nodes == [workertestnode] + + nodeclass = self.retmoteworker.nodespace.lib.get_node_by_id("workertestnode") + self.assertEqual(nodeclass.node_name, "workertestnode") + + async def test_external_worker_nodes_multiple_updates(self): + worker = self.retmoteworker.add_local_worker( + ExternalWorkerWithNodeShelves, "test_external_worker_nodes_multiple" + ) + + for _ in range(2): + print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) + worker.emit("nodes_update") + assert worker.get_nodeshelf() is not None + assert worker.get_nodeshelf().name == "test" + assert worker.get_nodeshelf().nodes == [workertestnode] + + assert isinstance(worker.nodeshelf, ref) + assert worker.nodeshelf() is not None + assert worker.nodeshelf().name == "test" + assert worker.nodeshelf().nodes == [workertestnode] + + print( + json.dumps(self.retmoteworker.nodespace.lib.full_serialize(), indent=4) + ) + print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) + + nodeclass = self.retmoteworker.nodespace.lib.get_node_by_id( + "workertestnode" + ) + self.assertEqual(nodeclass.node_name, "workertestnode") + + +# diff --git a/tests/test_loops.py b/tests/test_loops.py index 749f6d0..6bdd3d6 100644 --- a/tests/test_loops.py +++ b/tests/test_loops.py @@ -6,11 +6,8 @@ CustomLoop, LoopManager, ) -from funcnodes_core.testing import ( - set_in_test as fn_set_in_test, -) -fn_set_in_test() +from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown class _TestLoop(CustomLoop): @@ -19,6 +16,12 @@ async def loop(self): class TestCustomLoop(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + fn_setup() + + def tearDown(self) -> None: + fn_teardown() + async def asyncSetUp(self): self.logger = logging.getLogger("TestLogger") self.loop = _TestLoop(delay=0.2, logger=self.logger) @@ -84,6 +87,12 @@ async def test_continuous_run_calls_loop(self): class TestLoopManager(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + fn_setup() + + def tearDown(self) -> None: + fn_teardown() + async def asyncSetUp(self): self.worker = Mock() self.worker.logger = logging.getLogger("TestWorkerLogger") diff --git a/tests/test_socketworker.py b/tests/test_socketworker.py index 6b711b6..ee2503a 100644 --- a/tests/test_socketworker.py +++ b/tests/test_socketworker.py @@ -2,15 +2,11 @@ import asyncio from unittest.mock import AsyncMock from funcnodes_worker import SocketWorker -from funcnodes_core.testing import ( - teardown as fn_teardown, - set_in_test as fn_set_in_test, -) +from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown class TestSocketWorker(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): - fn_set_in_test() self.worker = SocketWorker(host="127.0.0.1", port=9382) self.worker.socket_loop._assert_connection = AsyncMock() self.worker.socket_loop.stop = AsyncMock() @@ -20,6 +16,10 @@ async def asyncTearDown(self): self.worker.stop() await asyncio.sleep(0.4) + def setUp(self) -> None: + fn_setup() + + def tearDown(self) -> None: fn_teardown() async def test_initial_state(self): diff --git a/tests/test_worker.py b/tests/test_worker.py index a3c6491..38a187e 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1,29 +1,27 @@ -from unittest import IsolatedAsyncioTestCase, TestCase -import funcnodes_core as fn -from funcnodes_worker import Worker -from funcnodes_worker.worker import WorkerState, NodeViewState -import tempfile +import asyncio +import io +import json +import logging import os from pathlib import Path -import asyncio import time -import json +import zipfile from copy import deepcopy -import logging -import threading +from typing import ClassVar, Type, Union -from funcnodes_core.testing import ( - teardown as fn_teardown, - set_in_test as fn_set_in_test, -) +import pytest +import funcnodes_core as fn + +from funcnodes_worker import Worker, FuncNodesExternalWorker, ExternalWorkerConfig +from funcnodes_worker.worker import WorkerState, NodeViewState +from pydantic import Field + + +from pytest_funcnodes import funcnodes_test class _TestWorkerClass(Worker): - def _on_nodespaceerror( - self, - error: Exception, - src: fn.NodeSpace, - ): + def _on_nodespaceerror(self, error: Exception, src: fn.NodeSpace): """handle nodespace errors""" def on_nodespaceevent(self, event, **kwargs): @@ -40,482 +38,642 @@ def testnode(a: int = 1) -> int: ) -class TestWorkerInitCases(TestCase): - Workerclass = _TestWorkerClass - workerkwargs = {} - - def setUp(self): - fn_set_in_test() - self.tempdir = tempfile.TemporaryDirectory() - self.workerkwargs["uuid"] = "testuuid" - self.worker = None - - async def asyncTearDown(self): - if self.worker: - self.worker.stop() - await asyncio.sleep(0.4) - del self.worker - fn_teardown() - - self.tempdir.cleanup() +@pytest.fixture +def worker_class(): + return _TestWorkerClass - def test_initialization(self): - self.worker = self.Workerclass(**self.workerkwargs) - self.assertIsInstance(self.worker, self.Workerclass) - def test_with_default_nodes(self): - self.worker = self.Workerclass(**self.workerkwargs, default_nodes=[testshelf]) - self.assertIsInstance(self.worker, self.Workerclass) +@pytest.fixture +def worker_kwargs(): + return {"uuid": "testuuid"} - def test_with_debug(self): - self.worker = self.Workerclass(**self.workerkwargs, debug=True) - self.assertIsInstance(self.worker, self.Workerclass) - self.assertEqual(self.worker.logger.level, logging.DEBUG) - - def test_initandrun(self): - wpath = fn.config.get_config_dir() / "workers" +@pytest.fixture +async def worker_case(worker_class: Type[_TestWorkerClass], tmp_path: Union[Path, str]): + worker = worker_class( + data_path=tmp_path, + default_nodes=[testshelf], + debug=True, + uuid="TestWorkerCase_testuuid", + ) + worker.write_config() + try: + yield worker + finally: + worker.stop() + await asyncio.sleep(0.4) - olfiles = ( - os.listdir(fn.config.get_config_dir() / "workers") if wpath.exists() else [] - ) - runthread = threading.Thread( - target=self.Workerclass.init_and_run_forever, - kwargs=self.workerkwargs, - daemon=True, +@pytest.fixture +async def interacting_worker(running_test_worker: _TestWorkerClass): + worker = running_test_worker + node1 = worker.add_node("test_node") + node2 = worker.add_node("test_node") + await asyncio.sleep(0.5) + worker.add_edge(node1.uuid, "out", node2.uuid, "a") + await asyncio.sleep(0.5) + return worker, node1, node2 + + +@pytest.fixture +def worker_instance( + worker_class: Type[_TestWorkerClass], + # tmp_path:Union[Path, str], + funcnodes_test_setup_teardown, +): + worker = worker_class( + # data_path=tmp_path, + default_nodes=[testshelf], + uuid="TestExternalWorkerUpdate", + ) + assert len(list(worker.data_path.parent.iterdir())) == 1, ( + f"data_path {worker.data_path.parent} is not empty, but {list(worker.data_path.parent.iterdir())}" + ) + return worker + + +@pytest.fixture +async def running_test_worker(worker_instance: _TestWorkerClass): + thread = worker_instance.run_forever_threaded() + await worker_instance.wait_for_running(timeout=10) + try: + yield worker_instance + finally: + worker_instance.stop() + thread.join() + + +@pytest.fixture(scope="function", autouse=True) +def register_ndoe(): + fn.node.register_node(testnode) + + +def create_test_node(worker): + node = worker.add_node("test_node") + assert isinstance(node, fn.Node) + assert isinstance(node, testnode) + retrieved = worker.get_node(node.uuid) + assert retrieved is node + return node + + +@funcnodes_test +def test_worker_initialization(worker_class, worker_kwargs): + worker = worker_class(**worker_kwargs) + try: + assert isinstance(worker, worker_class) + finally: + worker.stop() + + +@funcnodes_test +def test_with_default_nodes(worker_class, worker_kwargs): + worker = worker_class(**worker_kwargs, default_nodes=[testshelf]) + try: + assert isinstance(worker, worker_class) + finally: + worker.stop() + + +@funcnodes_test +def test_with_debug(worker_class: Type[_TestWorkerClass], worker_kwargs): + worker = worker_class(**worker_kwargs, debug=True) + try: + assert isinstance(worker, worker_class) + assert worker.logger.level == logging.DEBUG + finally: + worker.stop() + + +@funcnodes_test(disable_file_handler=False) +def test_worker_logger(worker_instance: _TestWorkerClass): + worker = worker_instance + assert worker.logger.level == logging.DEBUG + assert worker.logger.name == "funcnodes." + worker.uuid() + assert len(worker.logger.handlers) == 2, worker.logger.handlers + # At least one stream-like handler + file_handlers = [ + h for h in worker.logger.handlers if isinstance(h, logging.FileHandler) + ] + stream_handlers = [ + h + for h in worker.logger.handlers + if isinstance(h, logging.StreamHandler) + and not isinstance(h, logging.FileHandler) # exclude FileHandler + subclasses + ] + # One file-like handler (FileHandler / RotatingFileHandler / etc.) + assert len(file_handlers) == 1, file_handlers + + # One pure stream handler (console) + assert len(stream_handlers) == 1, stream_handlers + + +@funcnodes_test(disable_file_handler=False) +def test_initandrun(running_test_worker: _TestWorkerClass): + workersdir = fn.config.get_config_dir() / "workers" + worker = running_test_worker + workerdir = workersdir / f"worker_{worker.uuid()}" + worker_p_file = workersdir / f"worker_{worker.uuid()}.p" + + for _ in range(200): + if worker_p_file.exists(): + break + time.sleep(0.1) + time.sleep(2) + + newfiles = os.listdir(fn.config.get_config_dir() / "workers") + # newfiles = set(newfiles) - set(existing_files) + + assert workerdir.is_dir() + assert worker_p_file.exists() + + assert f"worker_{worker.uuid()}.p" in newfiles, ( + f"worker_{worker.uuid()}.p not found in {fn.config.get_config_dir() / 'workers'}" + ) + assert f"worker_{worker.uuid()}.runstate" in newfiles, ( + f"worker_{worker.uuid()}.runstate not found in {fn.config.get_config_dir() / 'workers'}" + ) + assert f"worker_{worker.uuid()}" in newfiles, ( + f"worker_{worker.uuid()} not found in {fn.config.get_config_dir() / 'workers'}" + ) + + with open(worker_p_file, "r") as file_handle: + pid = file_handle.read() + + assert pid.isdigit(), pid + assert os.getpid() == int(pid) + + with open(worker_p_file, "w") as file_handle: + json.dump({"cmd": "stop_worker"}, file_handle) + + for _ in range(150): + if not worker_p_file.exists(): + break + time.sleep(0.1) + time.sleep(0.5) + log_contents = None + if worker_p_file.exists(): + assert f"funcnodes.{worker.uuid()}.log" in os.listdir(workerdir), ( + f"funcnodes.testuuid.log not found in {workerdir}" ) - runthread.start() - workerdir = fn.config.get_config_dir() / "workers" - worker_p_file = workerdir / f"worker_{self.workerkwargs['uuid']}.p" - - # wait max 10 seconds for the worker to start - for i in range(200): - if worker_p_file.exists(): - break - time.sleep(0.1) - time.sleep(2) - - workersdir = fn.config.get_config_dir() / "workers" - workerdir = workersdir / f"worker_{self.workerkwargs['uuid']}" - newfiles = os.listdir(fn.config.get_config_dir() / "workers") - newfiles = set(newfiles) - set(olfiles) - - assert f"worker_{self.workerkwargs['uuid']}.p" in newfiles - assert f"worker_{self.workerkwargs['uuid']}.runstate" in newfiles - assert f"worker_{self.workerkwargs['uuid']}" in newfiles - assert workerdir.is_dir() - assert worker_p_file.exists() - - stopcmd = {"cmd": "stop_worker"} - - with open(worker_p_file, "r") as f: - pid = f.read() - - self.assertTrue(pid.isdigit(), pid) - - self.assertEqual(os.getpid(), int(pid)) - - with open(worker_p_file, "w") as f: - json.dump(stopcmd, f) - - # wait max 5 seconds for the worker to stop - for i in range(150): - if not runthread.is_alive(): - break - time.sleep(0.1) - - log = None - if runthread.is_alive(): - self.assertTrue( - "funcnodes.testuuid.log" in os.listdir(workerdir), os.listdir(workerdir) - ) - with open(workerdir / "funcnodes.testuuid.log", "r") as f: - log = f.read() - - self.assertFalse(runthread.is_alive(), log) - - runthread.join() - - -class TestWorkerCase(IsolatedAsyncioTestCase): - Workerclass = _TestWorkerClass - - async def asyncSetUp(self): - fn_set_in_test() - self.tempdir = tempfile.TemporaryDirectory() - self.tempdir_path = Path(self.tempdir.name) - self.worker = self.Workerclass( - data_path=self.tempdir_path, - default_nodes=[testshelf], - debug=True, - uuid="TestWorkerCase_testuuid", - ) - self.worker.write_config() - - async def asyncTearDown(self): - self.worker.stop() - await asyncio.sleep(0.4) - fn_teardown() - self.tempdir.cleanup() - - def test_initialization(self): - self.assertIsInstance(self.worker, self.Workerclass) - self.assertTrue(hasattr(self.worker, "nodespace")) - self.assertTrue(hasattr(self.worker, "loop_manager")) - - def test_uuid(self): - self.assertIsInstance(self.worker.uuid(), str) - - def test_config_generation(self): - config = fn.JSONEncoder.apply_custom_encoding(self.worker.config) - - self.maxDiff = None - expected = { - "uuid": self.worker.uuid(), - "name": self.worker.name(), - "data_path": self.tempdir_path.absolute().resolve().as_posix(), - "package_dependencies": {}, - "pid": os.getpid(), - "type": self.Workerclass.__name__, - "env_path": None, - "update_on_startup": { - "funcnodes": True, - "funcnodes-core": True, - "funcnodes-worker": True, - }, - "worker_dependencies": {}, - } - self.assertEqual(config, expected) - - def test_exportable_config(self): - config = self.worker.exportable_config() - self.assertIsInstance(config, dict) - expected = { - "name": self.worker.name(), - "package_dependencies": {}, - "type": self.Workerclass.__name__, - "update_on_startup": { - "funcnodes": True, - "funcnodes-core": True, - "funcnodes-worker": True, - }, - "worker_dependencies": {}, - } - self.assertEqual(config, expected) - - def test_write_config(self): - config_path = self.worker._config_file - self.worker.write_config() - self.assertTrue(os.path.exists(config_path)) - - def test_load_config(self): - self.worker.write_config() - config = self.worker.load_config() - self.assertIsNotNone(config) - self.assertEqual(config["uuid"], self.worker.uuid()) - - def test_process_file_handling(self): - self.worker._write_process_file() - process_file = self.worker._process_file - self.assertTrue(os.path.exists(process_file)) - - def test_save_state(self): - self.worker.save() - state_path = self.worker.local_nodespace - self.assertTrue(os.path.exists(state_path)) - - async def test_run_cmd(self): - cmd = {"cmd": "uuid", "kwargs": {}} - result = await self.worker.run_cmd(cmd) - self.assertEqual(result, self.worker.uuid()) - - async def test_full_state(self): - ser = fn.JSONEncoder.apply_custom_encoding(self.worker.full_state()) - self.assertIsInstance(ser, dict) - expected = { - "backend": { - "nodes": [], - "prop": {}, - "lib": { - "shelves": [ - { - "nodes": [ - { - "node_id": "test_node", - "inputs": [ - { - "type": "int", - "description": None, - "uuid": "a", - } - ], - "outputs": [ - { - "type": "int", - "description": None, - "uuid": "out", - } - ], - "description": "", - "node_name": "testnode", - } - ], - "subshelves": [], - "name": "testshelf", - "description": "Test shelf", - } - ] - }, - "edges": [], + with open(workerdir / f"funcnodes.{worker.uuid()}.log", "r") as logfile: + log_contents = logfile.read() + + assert not worker_p_file.exists(), log_contents + + +@funcnodes_test +async def test_worker_case_initialization(worker_case, worker_class): + assert isinstance(worker_case, worker_class) + assert hasattr(worker_case, "nodespace") + assert hasattr(worker_case, "loop_manager") + assert worker_case.nodespace.lib.has_node_id("test_node") + + +@funcnodes_test +async def test_worker_case_uuid(worker_case): + assert isinstance(worker_case.uuid(), str) + + +@funcnodes_test +async def test_worker_case_config_generation(worker_case): + config = fn.JSONEncoder.apply_custom_encoding(worker_case.config) + expected = { + "uuid": worker_case.uuid(), + "name": worker_case.name(), + "data_path": worker_case.data_path.absolute().resolve().as_posix(), + "package_dependencies": {}, + "pid": os.getpid(), + "type": worker_case.__class__.__name__, + "env_path": None, + "update_on_startup": { + "funcnodes": True, + "funcnodes-core": True, + "funcnodes-worker": True, + }, + "worker_dependencies": {}, + } + assert config == expected + + +@funcnodes_test +async def test_worker_case_exportable_config(worker_case): + config = worker_case.exportable_config() + expected = { + "name": worker_case.name(), + "package_dependencies": {}, + "type": worker_case.__class__.__name__, + "update_on_startup": { + "funcnodes": True, + "funcnodes-core": True, + "funcnodes-worker": True, + }, + "worker_dependencies": {}, + } + assert config == expected + + +@funcnodes_test +async def test_worker_case_write_config(worker_case): + config_path = worker_case._config_file + worker_case.write_config() + assert os.path.exists(config_path) + + +@funcnodes_test +async def test_worker_case_load_config(worker_case): + worker_case.write_config() + config = worker_case.load_config() + assert config is not None + assert config["uuid"] == worker_case.uuid() + + +@funcnodes_test +async def test_worker_case_process_file_handling(worker_case): + worker_case._write_process_file() + process_file = worker_case._process_file + assert os.path.exists(process_file) + + +@funcnodes_test +async def test_worker_case_save_state(worker_case): + worker_case.save() + assert os.path.exists(worker_case.local_nodespace) + + +@funcnodes_test +async def test_worker_run_cmd(worker_case): + cmd = {"cmd": "uuid", "kwargs": {}} + result = await worker_case.run_cmd(cmd) + assert result == worker_case.uuid() + + +@funcnodes_test +async def test_worker_full_state(worker_case): + ser = fn.JSONEncoder.apply_custom_encoding(worker_case.full_state()) + expected = { + "backend": { + "nodes": [], + "prop": {}, + "lib": { + "shelves": [ + { + "nodes": [ + { + "node_id": "test_node", + "inputs": [ + { + "type": "int", + "description": None, + "uuid": "a", + } + ], + "outputs": [ + { + "type": "int", + "description": None, + "uuid": "out", + } + ], + "description": "", + "node_name": "testnode", + } + ], + "subshelves": [], + "name": "testshelf", + "description": "Test shelf", + } + ] }, - "worker": {}, - "worker_dependencies": [], - "progress_state": { - "message": "", - "status": "", - "progress": 0, - "blocking": False, - }, - "meta": {"id": self.worker.nodespace_id, "version": fn.__version__}, - } - - ser.pop("view", None) # because this differes on other installations - self.assertEqual(ser, expected) - - def test_add_node(self): - node = self._add_node() - self.assertIsInstance(node, fn.Node) - - def _add_node(self): - node_id = "test_node" - addednode = self.worker.add_node(node_id) - self.assertIsInstance(addednode, fn.Node) - self.assertIsInstance(addednode, testnode) - - node = self.worker.get_node(addednode.uuid) - self.assertIsNotNone(node) - self.assertEqual(node, addednode) - return node - - def test_remove_node(self): - node = self._add_node() - self.worker.get_node(node.uuid) - self.worker.remove_node(node.uuid) - with self.assertRaises(ValueError): - self.worker.get_node(node.uuid) - - def test_add_edge(self): - node1 = self._add_node() - node2 = self._add_node() - - self.worker.add_edge(node1.uuid, "out", node2.uuid, "a") - edges = self.worker.get_edges() - self.assertEqual(len(edges), 1) - self.assertEqual( - edges, - [ - ( - node1.uuid, - "out", - node2.uuid, - "a", - ) - ], - ) - - def test_remove_edge(self): - self.test_add_edge() - edge = self.worker.get_edges()[0] - self.worker.remove_edge(*edge) - self.assertEqual(len(self.worker.get_edges()), 0) - - def test_update_node(self): - node = self._add_node() - self.worker.update_node(node.uuid, {"name": "Updated Node"}) - node = self.worker.get_node(node.uuid) - self.assertEqual(node.name, "Updated Node") - - async def test_run(self): - asyncio.create_task(self.worker.run_forever_async()) - await self.worker.wait_for_running(timeout=10) - self.assertTrue(self.worker.loop_manager.running) - self.worker.stop() - self.assertFalse(self.worker.loop_manager.running) - - async def test_run_threaded(self): - runthread = self.worker.run_forever_threaded() - await self.worker.wait_for_running(timeout=10) - self.worker.stop() - runthread.join() - self.assertFalse(self.worker.loop_manager.running) - # t = time.time() - - async def test_unknown_cmd(self): - cmd = {"cmd": "unknown", "kwargs": {}} - with self.assertRaises(Worker.UnknownCmdException): - await self.worker.run_cmd(cmd) - - async def test_run_double(self): - t1 = asyncio.create_task(self.worker.run_forever_async()) - await self.worker.wait_for_running(timeout=10) - assert self.worker._process_file.exists() - - t2 = asyncio.create_task(self.worker.run_forever_async()) - with self.assertRaises(RuntimeError): - async with asyncio.timeout(10): - await t2 - - # t1 should still be running while t2 should be done - self.assertFalse(t1.done()) - self.assertTrue(t2.done()) - - self.worker.stop() - async with asyncio.timeout(5): - await t1 - - async def test_load(self): - asyncio.create_task(self.worker.run_forever_async()) - await self.worker.wait_for_running(timeout=10) - data = WorkerState( - backend={ - "nodes": [], - "prop": {}, - "lib": { - "shelves": [ - { - "nodes": [ - { - "node_id": "test_node", - "inputs": [ - { - "type": "int", - "description": None, - "uuid": "a", - } - ], - "outputs": [ - { - "type": "int", - "description": None, - "uuid": "out", - } - ], - "description": "", - "node_name": "testnode", - } - ], - "subshelves": [], - "name": "testshelf", - "description": "Test shelf", - } - ] - }, - "edges": [], + "edges": [], + }, + "worker": {}, + "worker_dependencies": [], + "progress_state": { + "message": "", + "status": "", + "progress": 0, + "blocking": False, + }, + "meta": {"id": worker_case.nodespace_id, "version": fn.__version__}, + } + + ser.pop("view", None) + assert ser == expected + + +@funcnodes_test +async def test_worker_add_node(worker_case): + node = create_test_node(worker_case) + assert isinstance(node, fn.Node) + + +@funcnodes_test +async def test_worker_remove_node(worker_case): + node = create_test_node(worker_case) + worker_case.remove_node(node.uuid) + with pytest.raises(ValueError): + worker_case.get_node(node.uuid) + + +@funcnodes_test +async def test_worker_add_edge(worker_case): + node1 = create_test_node(worker_case) + node2 = create_test_node(worker_case) + worker_case.add_edge(node1.uuid, "out", node2.uuid, "a") + edges = worker_case.get_edges() + assert len(edges) == 1 + assert edges == [(node1.uuid, "out", node2.uuid, "a")] + + +@funcnodes_test +async def test_worker_remove_edge(worker_case): + node1 = create_test_node(worker_case) + node2 = create_test_node(worker_case) + worker_case.add_edge(node1.uuid, "out", node2.uuid, "a") + edge = worker_case.get_edges()[0] + worker_case.remove_edge(*edge) + assert len(worker_case.get_edges()) == 0 + + +@funcnodes_test +async def test_worker_update_node(worker_case): + node = create_test_node(worker_case) + worker_case.update_node(node.uuid, {"name": "Updated Node"}) + updated = worker_case.get_node(node.uuid) + assert updated.name == "Updated Node" + + +@funcnodes_test +async def test_worker_run(worker_case): + task = asyncio.create_task(worker_case.run_forever_async()) + await worker_case.wait_for_running(timeout=10) + assert worker_case.loop_manager.running + worker_case.stop() + assert not worker_case.loop_manager.running + async with asyncio.timeout(5): + await task + + +@funcnodes_test +async def test_worker_run_threaded(worker_case): + runthread = worker_case.run_forever_threaded() + await worker_case.wait_for_running(timeout=10) + worker_case.stop() + runthread.join() + assert not worker_case.loop_manager.running + + +@funcnodes_test +async def test_worker_unknown_cmd(worker_case): + cmd = {"cmd": "unknown", "kwargs": {}} + with pytest.raises(Worker.UnknownCmdException): + await worker_case.run_cmd(cmd) + + +@funcnodes_test +async def test_worker_run_double(worker_case): + first_task = asyncio.create_task(worker_case.run_forever_async()) + await worker_case.wait_for_running(timeout=10) + assert worker_case._process_file.exists() + + second_task = asyncio.create_task(worker_case.run_forever_async()) + with pytest.raises(RuntimeError): + async with asyncio.timeout(10): + await second_task + + assert not first_task.done() + assert second_task.done() + + worker_case.stop() + async with asyncio.timeout(5): + await first_task + + +@funcnodes_test +async def test_worker_load(worker_case): + run_task = asyncio.create_task(worker_case.run_forever_async()) + await worker_case.wait_for_running(timeout=10) + data = WorkerState( + backend={ + "nodes": [], + "prop": {}, + "lib": { + "shelves": [ + { + "nodes": [ + { + "node_id": "test_node", + "inputs": [ + { + "type": "int", + "description": None, + "uuid": "a", + } + ], + "outputs": [ + { + "type": "int", + "description": None, + "uuid": "out", + } + ], + "description": "", + "node_name": "testnode", + } + ], + "subshelves": [], + "name": "testshelf", + "description": "Test shelf", + } + ] }, - view={}, - meta={}, - external_workers={}, - ) - - self.assertIsNotNone(self.worker.nodespace_loop) - self.assertIsNotNone(self.worker.loop_manager) - self.assertTrue(self.worker.loop_manager.running) - - self.assertIsNotNone(self.worker.nodespace_loop._manager) - await self.worker.load(data) - - _d = deepcopy(data) - _d["meta"]["id"] = "abc" - # should raise an id to short erroer - with self.assertRaises(ValueError): - await self.worker.load(_d) + "edges": [], + }, + view={}, + meta={}, + external_workers={}, + ) + + assert worker_case.nodespace_loop is not None + assert worker_case.loop_manager is not None + assert worker_case.loop_manager.running + assert worker_case.nodespace_loop._manager is not None + + await worker_case.load(data) + + mutated = deepcopy(data) + mutated["meta"]["id"] = "abc" + with pytest.raises(ValueError): + await worker_case.load(mutated) + + mutated = deepcopy(data) + mutated["meta"]["id"] = None + await worker_case.load(mutated) + + mutated = deepcopy(data) + mutated["meta"]["id"] = "a" * 32 + await worker_case.load(mutated) + + worker_case.stop() + async with asyncio.timeout(5): + await run_task - _d = deepcopy(data) - _d["meta"]["id"] = None - # this should work - await self.worker.load(_d) - - _d = deepcopy(data) - _d["meta"]["id"] = "a" * 32 - # this should work - await self.worker.load(_d) - - -class TestWorkerInteractingCase(IsolatedAsyncioTestCase): - Workerclass = _TestWorkerClass - - async def asyncSetUp(self): - self.tempdir = tempfile.TemporaryDirectory() - fn_set_in_test() - self.worker = self.Workerclass( - data_path=Path(self.tempdir.name), default_nodes=[testshelf], debug=True - ) - - asyncio.create_task(self.worker.run_forever_async()) - async with asyncio.timeout(10): - while self.worker.runstate != "running": - await asyncio.sleep(0.1) - await asyncio.sleep(0.5) - node_id = "test_node" - self.node1 = self.worker.add_node(node_id) - self.node2 = self.worker.add_node(node_id) - await asyncio.sleep(0.5) - self.worker.add_edge(self.node1.uuid, "out", self.node2.uuid, "a") - await asyncio.sleep(0.5) # let the nodes trigger - - async def asyncTearDown(self): - self.worker.stop() - await asyncio.sleep(0.4) - fn_teardown() - self.tempdir.cleanup() - - async def test_get_io_value(self): - # list nodes - nodes = self.worker.get_nodes() - self.assertEqual(len(nodes), 2) - - v = self.worker.get_io_value(self.node1.uuid, "out") - - self.assertEqual(v, 1) - - async def test_set_io_value(self): - self.worker.set_io_value(self.node1.uuid, "a", 2, set_default=True) - await asyncio.sleep(0.1) # let the nodes trigger - v = self.worker.get_io_value(self.node1.uuid, "out") - self.assertEqual(v, 2) - - async def test_update_node_view(self): - self.worker.update_node_view( - self.node1.uuid, - NodeViewState( - pos=(10, 10), - size=(100, 100), - ), - ) - vs = self.worker.view_state() - exp_nodes = {} - exp_nodes[self.node1.uuid] = { - "pos": (10, 10), - "size": (100, 100), - } - exp_nodes[self.node2.uuid] = { - "pos": (0, 0), - "size": (200, 250), - } - - self.assertEqual(vs["nodes"], exp_nodes) - - async def test_add_package_dependency(self): - await self.worker.add_package_dependency("funcnodes-basic") - self.assertIn("funcnodes-basic", self.worker._package_dependencies) - - async def test_upload(self): - data = b"hello" - self.worker.upload(data, "test.txt") - - self.assertTrue( - os.path.exists(os.path.join(self.worker.files_path, "test.txt")) - ) - with self.assertRaises(ValueError): - self.worker.upload(data, "../test.txt") + +@funcnodes_test +async def test_get_io_value(interacting_worker): + worker, node1, node2 = interacting_worker + nodes = worker.get_nodes() + assert len(nodes) == 2 + value = worker.get_io_value(node1.uuid, "out") + assert value == 1 + + +@funcnodes_test +async def test_set_io_value(interacting_worker): + worker, node1, _ = interacting_worker + worker.set_io_value(node1.uuid, "a", 2, set_default=True) + await asyncio.sleep(0.1) + value = worker.get_io_value(node1.uuid, "out") + assert value == 2 + + +@funcnodes_test +async def test_update_node_view(interacting_worker): + worker, node1, node2 = interacting_worker + worker.update_node_view( + node1.uuid, + NodeViewState( + pos=(10, 10), + size=(100, 100), + ), + ) + view_state = worker.view_state() + expected_nodes = { + node1.uuid: {"pos": (10, 10), "size": (100, 100)}, + node2.uuid: {"pos": (0, 0), "size": (200, 250)}, + } + assert view_state["nodes"] == expected_nodes + + +@funcnodes_test +async def test_add_package_dependency(interacting_worker): + worker, _, _ = interacting_worker + await worker.add_package_dependency("funcnodes-basic") + assert "funcnodes-basic" in worker._package_dependencies + + +@funcnodes_test +async def test_upload(interacting_worker): + worker, _, _ = interacting_worker + data = b"hello" + worker.upload(data, "test.txt") + assert os.path.exists(os.path.join(worker.files_path, "test.txt")) + with pytest.raises(ValueError): + worker.upload(data, "../test.txt") + + +class _CountingShelfConfig(ExternalWorkerConfig): + marker: int = 0 + + +class CountingShelfWorker(FuncNodesExternalWorker): + NODECLASSID = "test_counting_shelf_worker" + config_cls = _CountingShelfConfig + + def __init__(self, *args, **kwargs) -> None: + self.shelf_calls = 0 + self.last_marker = 0 + super().__init__(*args, **kwargs) + self.last_marker = self.config.marker + + async def loop(self): + await asyncio.sleep(0.01) + + def post_config_update(self): + self.last_marker = self.config.marker + self.emit("nodes_update") + + def get_nodeshelf(self): + self.shelf_calls += 1 + return None + + +class _SecretiveConfig(ExternalWorkerConfig): + EXPORT_EXCLUDE_FIELDS: ClassVar[set[str]] = {"class_hidden"} + + class_hidden: str = "secret-from-class" + field_hidden: str = Field( + default="secret-from-field", json_schema_extra={"export": False} + ) + visible: str = "visible" + + +class SecretiveWorker(FuncNodesExternalWorker): + NODECLASSID = "test_secretive_worker" + config_cls = _SecretiveConfig + + def get_nodeshelf(self): + return None + + +@funcnodes_test +async def test_export_worker_excludes_external_worker_sensitive_fields( + running_test_worker: _TestWorkerClass, +): + external_worker = running_test_worker + worker_instance = external_worker.add_local_worker( + SecretiveWorker, "secretive-worker" + ) + external_worker.update_external_worker( + worker_instance.uuid, + SecretiveWorker.NODECLASSID, + config={ + "class_hidden": "top-secret", + "field_hidden": "token-123", + "visible": "fine", + }, + ) + await asyncio.sleep(0.2) + + full_state = external_worker.get_save_state() + assert len(full_state["external_workers"][SecretiveWorker.NODECLASSID]) == 1, ( + full_state + ) + saved_config = full_state["external_workers"][SecretiveWorker.NODECLASSID][0][ + "config" + ] + assert saved_config["class_hidden"] == "top-secret" + assert saved_config["field_hidden"] == "token-123" + assert saved_config["visible"] == "fine" + + exported = external_worker.export_worker() + with zipfile.ZipFile(io.BytesIO(exported), "r") as zf: + exported_state = json.loads(zf.read("state").decode("utf-8")) + + exported_config = exported_state["external_workers"][SecretiveWorker.NODECLASSID][ + 0 + ]["config"] + assert "class_hidden" not in exported_config + assert "field_hidden" not in exported_config + assert exported_config["visible"] == "fine" + + +@funcnodes_test +async def test_update_external_worker_refreshes_shelf_without_event( + worker_instance: _TestWorkerClass, +): + ex_worker_instance = worker_instance.add_local_worker( + CountingShelfWorker, "counting-shelf-worker" + ) + assert ex_worker_instance.shelf_calls == 1 + + worker_instance.update_external_worker( + ex_worker_instance.uuid, + CountingShelfWorker.NODECLASSID, + config={"marker": 1}, + ) + + await asyncio.sleep(0.2) + + assert ex_worker_instance.shelf_calls == 2 diff --git a/tests/test_ws_worker.py b/tests/test_ws_worker.py index 61bf220..320ce3b 100644 --- a/tests/test_ws_worker.py +++ b/tests/test_ws_worker.py @@ -1,11 +1,7 @@ import asyncio import time from unittest import IsolatedAsyncioTestCase -from funcnodes_core.testing import ( - set_in_test as fn_set_in_test, -) - -fn_set_in_test() +from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown from funcnodes_worker import ( # noqa: E402 WSWorker, @@ -16,6 +12,12 @@ if aiohttp: class TestWSWorker(IsolatedAsyncioTestCase): + def setUp(self) -> None: + fn_setup() + + def tearDown(self) -> None: + fn_teardown() + async def test_ws_worker(self): ws_worker = WSWorker() @@ -50,6 +52,12 @@ async def listentask(): else: class TestPlaceholder(IsolatedAsyncioTestCase): + def setUp(self) -> None: + fn_setup() + + def tearDown(self) -> None: + fn_teardown() + async def test_placeholder(self): with self.assertRaises(DependencyError): WSWorker() From d6a77e5369b8daf569a7c32aaecfc50dbc9e4b3c Mon Sep 17 00:00:00 2001 From: Julian Kimmig Date: Thu, 27 Nov 2025 19:52:47 +0100 Subject: [PATCH 10/19] =?UTF-8?q?bump:=20version=201.2.1=20=E2=86=92=201.3?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 15 +++++++++++++++ pyproject.toml | 4 ++-- uv.lock | 12 ++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5e2bee..41f678c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 1.3.0 (2025-11-27) + +### Feat + +- **external-worker**: support exportable configs and nodeshelf updates +- **websocket**: implement graceful client connection closure and enhance message enqueue handling +- **worker**: enhance FuncNodesExternalWorker with nodeshelf property and logging for configuration updates +- **external_worker**: introduce ExternalWorkerConfig for improved configuration management and update FuncNodesExternalWorker to utilize it + +### Fix + +- **worker**: align external worker shelf updates and export +- **loop**: handle closed or missing event loop for tasks +- **tests**: correct asyncio_default_fixture_loop_scope format in pytest configuration + ## 1.2.1 (2025-11-06) ### Fix diff --git a/pyproject.toml b/pyproject.toml index 1403ba0..b5bd0f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "funcnodes-worker" -version = "1.2.1" +version = "1.3.0" description = "Worker package for FuncNodes" readme = "README.md" authors = [{name = "Julian Kimmig", email = "julian.kimmig@linkdlab.de"}] @@ -8,7 +8,7 @@ authors = [{name = "Julian Kimmig", email = "julian.kimmig@linkdlab.de"}] requires-python = ">=3.11" dependencies = [ "asynctoolkit>=0.1.1", - "funcnodes-core>=2.0.0a3", + "funcnodes-core>=2.0.0", "packaging>=24.2", "pip>=25.0.1", "pydantic>=2.12.4", diff --git a/uv.lock b/uv.lock index 1be2deb..98192c5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "sys_platform != 'emscripten'", @@ -429,7 +429,7 @@ wheels = [ [[package]] name = "funcnodes-core" -version = "2.0.0a3" +version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, @@ -439,9 +439,9 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/02/683c1e20132b9d865b747ada8829c279c759b15ac25d8e289a797515e61a/funcnodes_core-2.0.0a3.tar.gz", hash = "sha256:1c8250d7cfb81a3b70bf47e552da5ea4e22dbbbcc11c0f439fbc56046aeb6108", size = 189872, upload-time = "2025-11-27T11:11:28.071Z" } +sdist = { url = "https://files.pythonhosted.org/packages/67/44/6ddee77de599ede9fe598106bec950790838103370fe812efe0548587d6b/funcnodes_core-2.0.0.tar.gz", hash = "sha256:73d5a9731b2d30ad98fe9c47d809de8a324dafcdbbc3701c86194c0d1b535b52", size = 190475, upload-time = "2025-11-27T18:51:11.954Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/ca/ca0ba069573e33d4831ab9789c7f90978959a8265666063dd0b69db5fac6/funcnodes_core-2.0.0a3-py3-none-any.whl", hash = "sha256:11ab6c16eba70a973061f291f1eb5ca4b66ecda71ea60d739f5131a4b96bde91", size = 91647, upload-time = "2025-11-27T11:11:26.728Z" }, + { url = "https://files.pythonhosted.org/packages/49/bf/5f6652700e98a856e31bd80b608ca89c8f2b558c2a40c980c3cb39eefc16/funcnodes_core-2.0.0-py3-none-any.whl", hash = "sha256:6e42ef5065162adf09f3ffe2fff0fc5d1397235e98c670bdc8bf031bd0883e8d", size = 91624, upload-time = "2025-11-27T18:51:10.504Z" }, ] [[package]] @@ -474,7 +474,7 @@ wheels = [ [[package]] name = "funcnodes-worker" -version = "1.2.1" +version = "1.3.0" source = { editable = "." } dependencies = [ { name = "asynctoolkit" }, @@ -524,7 +524,7 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'http'" }, { name = "aiohttp-cors", marker = "extra == 'http'" }, { name = "asynctoolkit", specifier = ">=0.1.1" }, - { name = "funcnodes-core", specifier = ">=2.0.0a3" }, + { name = "funcnodes-core", specifier = ">=2.0.0" }, { name = "funcnodes-worker", extras = ["venv", "http", "subprocess-monitor"], marker = "extra == 'all'" }, { name = "packaging", specifier = ">=24.2" }, { name = "pip", specifier = ">=25.0.1" }, From 8c1f30dd2e9fac24f7e5f4567df1fdf3fbd5b99d Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Fri, 28 Nov 2025 09:23:30 +0100 Subject: [PATCH 11/19] refactor(tests): update worker fixture to use test name for UUID Modify worker_kwargs and worker_case fixtures to dynamically generate UUIDs based on the test name. Adjust test assertions and remove unnecessary try-finally blocks for cleaner code. Update test decorators to support no prefix for specific tests. --- tests/test_worker.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/test_worker.py b/tests/test_worker.py index 38a187e..848fb1d 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -44,17 +44,21 @@ def worker_class(): @pytest.fixture -def worker_kwargs(): - return {"uuid": "testuuid"} +def worker_kwargs(request: pytest.FixtureRequest): + return {"uuid": request.node.name} @pytest.fixture -async def worker_case(worker_class: Type[_TestWorkerClass], tmp_path: Union[Path, str]): +async def worker_case( + worker_class: Type[_TestWorkerClass], + tmp_path: Union[Path, str], + request: pytest.FixtureRequest, +): worker = worker_class( data_path=tmp_path, default_nodes=[testshelf], debug=True, - uuid="TestWorkerCase_testuuid", + uuid=f"TestWorkerCase_{request.node.name}", ) worker.write_config() try: @@ -80,11 +84,13 @@ def worker_instance( worker_class: Type[_TestWorkerClass], # tmp_path:Union[Path, str], funcnodes_test_setup_teardown, + # get test name + request: pytest.FixtureRequest, ): worker = worker_class( # data_path=tmp_path, default_nodes=[testshelf], - uuid="TestExternalWorkerUpdate", + uuid=request.node.name, ) assert len(list(worker.data_path.parent.iterdir())) == 1, ( f"data_path {worker.data_path.parent} is not empty, but {list(worker.data_path.parent.iterdir())}" @@ -120,13 +126,10 @@ def create_test_node(worker): @funcnodes_test def test_worker_initialization(worker_class, worker_kwargs): worker = worker_class(**worker_kwargs) - try: - assert isinstance(worker, worker_class) - finally: - worker.stop() + assert isinstance(worker, worker_class) -@funcnodes_test +@funcnodes_test(no_prefix=True) def test_with_default_nodes(worker_class, worker_kwargs): worker = worker_class(**worker_kwargs, default_nodes=[testshelf]) try: @@ -135,7 +138,7 @@ def test_with_default_nodes(worker_class, worker_kwargs): worker.stop() -@funcnodes_test +@funcnodes_test(no_prefix=True) def test_with_debug(worker_class: Type[_TestWorkerClass], worker_kwargs): worker = worker_class(**worker_kwargs, debug=True) try: @@ -214,7 +217,7 @@ def test_initandrun(running_test_worker: _TestWorkerClass): log_contents = None if worker_p_file.exists(): assert f"funcnodes.{worker.uuid()}.log" in os.listdir(workerdir), ( - f"funcnodes.testuuid.log not found in {workerdir}" + f"funcnodes.{worker.uuid()}.log not found in {workerdir}" ) with open(workerdir / f"funcnodes.{worker.uuid()}.log", "r") as logfile: log_contents = logfile.read() From 6bce27a5173a555571bf0b73957ceff9012f8a00 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Fri, 28 Nov 2025 09:23:46 +0100 Subject: [PATCH 12/19] refactor(tests): migrate test_client_connection to pytest and improve test structure Convert tests in test_client_connection.py from unittest to pytest, utilizing fixtures for logger setup. Enhance test readability and maintainability by removing the IsolatedAsyncioTestCase class and directly defining async test functions. --- tests/test_client_connection.py | 45 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/test_client_connection.py b/tests/test_client_connection.py index bfd9ab6..297a599 100644 --- a/tests/test_client_connection.py +++ b/tests/test_client_connection.py @@ -1,10 +1,14 @@ import asyncio import logging -from unittest import IsolatedAsyncioTestCase + +import pytest from funcnodes_worker.websocket import ClientConnection +pytestmark = pytest.mark.asyncio + + class DummyWebSocket: def __init__(self, delay: float = 0.0): self.delay = delay @@ -23,26 +27,31 @@ async def close(self, *_, **__): self.closed = True -class TestClientConnection(IsolatedAsyncioTestCase): - async def test_close_cancels_send_task(self): - ws = DummyWebSocket(delay=0.1) - client = ClientConnection(ws, logging.getLogger("test_close")) +@pytest.fixture +def logger(): + return logging.getLogger("test_client_connection") + + +async def test_close_cancels_send_task(logger): + ws = DummyWebSocket(delay=0.1) + client = ClientConnection(ws, logger) + + # Enqueue data so the send loop is actively processing. + await client.enqueue("ping") + await asyncio.sleep(0.01) - # Enqueue data so the send loop is actively processing. - await client.enqueue("ping") - await asyncio.sleep(0.01) + await client.close() - await client.close() + assert client.send_task.done() + assert client.queue.empty() - self.assertTrue(client.send_task.done()) - self.assertTrue(client.queue.empty()) - async def test_enqueue_after_close_is_noop(self): - ws = DummyWebSocket() - client = ClientConnection(ws, logging.getLogger("test_enqueue")) +async def test_enqueue_after_close_is_noop(logger): + ws = DummyWebSocket() + client = ClientConnection(ws, logger) - await client.close() - await client.enqueue("ignored") + await client.close() + await client.enqueue("ignored") - self.assertTrue(client.send_task.done()) - self.assertFalse(ws.sent) + assert client.send_task.done() + assert ws.sent == [] From c6e51dbb9a08800a9edb1f9d19c564ff5a65c37f Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Fri, 28 Nov 2025 09:28:02 +0100 Subject: [PATCH 13/19] refactor(tests): migrate SocketWorker tests to pytest and utilize fixtures Transform tests in test_socketworker.py from unittest to pytest, implementing fixtures for setup and teardown. This change enhances test organization and readability by removing the IsolatedAsyncioTestCase class and defining async test functions directly. Additionally, the use of pytest-funcnodes decorators streamlines the test execution process. --- tests/test_socketworker.py | 103 +++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/tests/test_socketworker.py b/tests/test_socketworker.py index ee2503a..4e8d0ef 100644 --- a/tests/test_socketworker.py +++ b/tests/test_socketworker.py @@ -1,53 +1,56 @@ -import unittest import asyncio from unittest.mock import AsyncMock + +import pytest + from funcnodes_worker import SocketWorker -from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown - - -class TestSocketWorker(unittest.IsolatedAsyncioTestCase): - async def asyncSetUp(self): - self.worker = SocketWorker(host="127.0.0.1", port=9382) - self.worker.socket_loop._assert_connection = AsyncMock() - self.worker.socket_loop.stop = AsyncMock() - - async def asyncTearDown(self): - if self.worker: - self.worker.stop() - await asyncio.sleep(0.4) - - def setUp(self) -> None: - fn_setup() - - def tearDown(self) -> None: - fn_teardown() - - async def test_initial_state(self): - self.assertEqual(self.worker.socket_loop._host, "127.0.0.1") - self.assertEqual(self.worker.socket_loop._port, 9382) - - async def test_send_message(self): - writer = AsyncMock() - await self.worker.sendmessage("test message", writer=writer) - writer.write.assert_called() - writer.drain.assert_called() - - async def test_send_message_to_clients(self): - writer1 = AsyncMock() - writer2 = AsyncMock() - self.worker.socket_loop.clients = [writer1, writer2] - await self.worker.sendmessage("test message") - writer1.write.assert_called() - writer2.write.assert_called() - - async def test_stop(self): - asyncio.create_task(self.worker.run_forever_async()) - await self.worker.wait_for_running(timeout=10) - await asyncio.sleep(1) - self.assertTrue(self.worker.socket_loop.running) - self.worker.stop() - self.assertFalse(self.worker.socket_loop.running) - - -if __name__ == "__main__": - unittest.main() +from pytest_funcnodes import funcnodes_test + + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def worker(): + worker = SocketWorker(host="127.0.0.1", port=9382) + worker.socket_loop._assert_connection = AsyncMock() + worker.socket_loop.stop = AsyncMock() + try: + yield worker + finally: + worker.stop() + await asyncio.sleep(0.4) + + +@funcnodes_test(no_prefix=True) +async def test_initial_state(worker): + assert worker.socket_loop._host == "127.0.0.1" + assert worker.socket_loop._port == 9382 + + +@funcnodes_test(no_prefix=True) +async def test_send_message(worker): + writer = AsyncMock() + await worker.sendmessage("test message", writer=writer) + writer.write.assert_called() + writer.drain.assert_called() + + +@funcnodes_test(no_prefix=True) +async def test_send_message_to_clients(worker): + writer1 = AsyncMock() + writer2 = AsyncMock() + worker.socket_loop.clients = [writer1, writer2] + await worker.sendmessage("test message") + writer1.write.assert_called() + writer2.write.assert_called() + + +@funcnodes_test(no_prefix=True) +async def test_stop(worker): + asyncio.create_task(worker.run_forever_async()) + await worker.wait_for_running(timeout=10) + await asyncio.sleep(1) + assert worker.socket_loop.running + worker.stop() + assert not worker.socket_loop.running From d3a89d1003d9991c57c51743d0bf69b007fcdb47 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Fri, 28 Nov 2025 09:28:39 +0100 Subject: [PATCH 14/19] refactor(tests): remove unused pytestmark from test_socketworker.py Eliminate the unused pytestmark variable from test_socketworker.py to clean up the code and improve clarity. This change contributes to maintaining a tidy test suite as part of the ongoing migration to pytest. --- tests/test_socketworker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_socketworker.py b/tests/test_socketworker.py index 4e8d0ef..9ab6166 100644 --- a/tests/test_socketworker.py +++ b/tests/test_socketworker.py @@ -7,9 +7,6 @@ from pytest_funcnodes import funcnodes_test -pytestmark = pytest.mark.asyncio - - @pytest.fixture async def worker(): worker = SocketWorker(host="127.0.0.1", port=9382) From 7b371dcacfa6f631940ea8789470e03fdd841185 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Sat, 29 Nov 2025 23:21:51 +0100 Subject: [PATCH 15/19] refactor(tests): migrate WSWorker tests to pytest and implement fixtures Transform tests in test_ws_worker.py from unittest to pytest, utilizing fixtures for setup and teardown. This change enhances test organization and readability by removing the IsolatedAsyncioTestCase class and defining async test functions directly. The use of pytest-funcnodes decorators further streamlines the test execution process. --- tests/test_ws_worker.py | 92 ++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/tests/test_ws_worker.py b/tests/test_ws_worker.py index 320ce3b..2d7afae 100644 --- a/tests/test_ws_worker.py +++ b/tests/test_ws_worker.py @@ -1,63 +1,51 @@ import asyncio import time -from unittest import IsolatedAsyncioTestCase -from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown -from funcnodes_worker import ( # noqa: E402 - WSWorker, -) +import pytest + +from funcnodes_worker import WSWorker # noqa: E402 from funcnodes_worker._opts import aiohttp, DependencyError # noqa: E402 +from pytest_funcnodes import funcnodes_test if aiohttp: - class TestWSWorker(IsolatedAsyncioTestCase): - def setUp(self) -> None: - fn_setup() - - def tearDown(self) -> None: - fn_teardown() - - async def test_ws_worker(self): - ws_worker = WSWorker() - - ws_worker.run_forever_threaded() - - port = ws_worker.port - host = ws_worker.host - - # make a connection to the websocket server - MAXTIME = 10 - async with aiohttp.ClientSession() as session: - async with session.ws_connect(f"ws://{host}:{port}") as ws: - - async def listentask(): - async for msg in ws: - print(msg) - - await ws.send_json({"type": "cmd", "cmd": "stop_worker"}) - asyncio.create_task(listentask()) - - stime = time.time() - self.assertFalse(ws.closed) - while not ws.closed and time.time() - stime < MAXTIME: - await asyncio.sleep( - 0.5 - ) # Poll until the connection is fully closed - self.assertTrue(ws.closed) - - # Wait for WebSocket to fully close - + @pytest.fixture + def ws_worker(): + worker = WSWorker() + thread = worker.run_forever_threaded() + try: + yield worker + finally: + worker.stop() + thread.join() + + @funcnodes_test(no_prefix=True) + async def test_ws_worker(ws_worker): + port = ws_worker.port + host = ws_worker.host + await asyncio.sleep(1) + max_time = 10 + async with aiohttp.ClientSession() as session: + async with session.ws_connect(f"ws://{host}:{port}") as ws: + + async def listentask(): + async for msg in ws: + print(msg) + + await ws.send_json({"type": "cmd", "cmd": "stop_worker"}) + asyncio.create_task(listentask()) + + start = time.time() + assert not ws.closed + while not ws.closed and time.time() - start < max_time: + await asyncio.sleep(0.5) + + assert ws.closed else: - class TestPlaceholder(IsolatedAsyncioTestCase): - def setUp(self) -> None: - fn_setup() - - def tearDown(self) -> None: - fn_teardown() - - async def test_placeholder(self): - with self.assertRaises(DependencyError): - WSWorker() + @funcnodes_test(no_prefix=True) + async def test_placeholder(): + with pytest.raises(DependencyError): + WSWorker() From 65cff4db246b3d7c659632032d3ef526949e4477 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Sat, 29 Nov 2025 23:31:30 +0100 Subject: [PATCH 16/19] refactor(tests): migrate test_loops to pytest and implement fixtures Transform tests in test_loops.py from unittest to pytest, utilizing fixtures for setup and teardown. This change enhances test organization and readability by removing the IsolatedAsyncioTestCase class and defining async test functions directly. The use of pytest-funcnodes decorators further streamlines the test execution process. --- tests/test_loops.py | 291 ++++++++++++++++++++++++-------------------- 1 file changed, 160 insertions(+), 131 deletions(-) diff --git a/tests/test_loops.py b/tests/test_loops.py index 6bdd3d6..8ab12a6 100644 --- a/tests/test_loops.py +++ b/tests/test_loops.py @@ -1,13 +1,18 @@ -import unittest import asyncio import logging +from contextlib import suppress from unittest.mock import AsyncMock, Mock + +import pytest + from funcnodes_worker.loop import ( CustomLoop, LoopManager, ) -from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown +from pytest_funcnodes import funcnodes_test + +pytestmark = pytest.mark.asyncio class _TestLoop(CustomLoop): @@ -15,132 +20,156 @@ async def loop(self): pass -class TestCustomLoop(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - fn_setup() - - def tearDown(self) -> None: - fn_teardown() - - async def asyncSetUp(self): - self.logger = logging.getLogger("TestLogger") - self.loop = _TestLoop(delay=0.2, logger=self.logger) - self.loop.loop = AsyncMock() - - async def test_initial_state(self): - self.assertFalse(self.loop.running) - self.assertFalse(self.loop.stopped) - self.assertIsNone(self.loop.manager) - - async def test_manager_assignment(self): - mock_manager = Mock() - self.loop.manager = mock_manager - self.assertEqual(self.loop.manager, mock_manager) - - async def test_manager_reassignment_fails(self): - mock_manager = Mock() - self.loop.manager = mock_manager - with self.assertRaises(ValueError): - self.loop.manager = Mock() - - async def test_stop(self): - self.loop._running = True - await self.loop.stop() - self.assertFalse(self.loop.running) - self.assertTrue(self.loop.stopped) - - async def test_pause(self): - class CountingLoop(CustomLoop): - counter = 0 - - async def loop(self): - self.counter += 1 - - loop = CountingLoop() - asyncio.create_task(loop.continuous_run()) - await asyncio.sleep(0.3) - self.assertGreater(loop.counter, 1) - loop.pause() - fixed_counter = loop.counter - await asyncio.sleep(0.3) - self.assertEqual(loop.counter, fixed_counter) - loop.resume() - await asyncio.sleep(0.3) - self.assertGreater(loop.counter, fixed_counter) - loop.pause() - fixed_counter = loop.counter - await asyncio.sleep(0.3) - self.assertEqual(loop.counter, fixed_counter) - loop.resume_in(1) - await asyncio.sleep(0.5) - self.assertEqual(loop.counter, fixed_counter) - await asyncio.sleep(1) - self.assertGreater(loop.counter, fixed_counter) - - async def test_continuous_run_calls_loop(self): - self.loop._running = True - task = asyncio.create_task(self.loop.continuous_run()) - await asyncio.sleep(0.3) # Let it run for a while - self.loop._running = False # Stop the loop - task.cancel() - self.loop.loop.assert_called() - - -class TestLoopManager(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - fn_setup() - - def tearDown(self) -> None: - fn_teardown() - - async def asyncSetUp(self): - self.worker = Mock() - self.worker.logger = logging.getLogger("TestWorkerLogger") - self.manager = LoopManager(self.worker) - self.custom_loop = AsyncMock(spec=CustomLoop) - - async def test_add_loop_while_stopped(self): - self.manager.add_loop(self.custom_loop) - self.assertIn(self.custom_loop, self.manager._loops_to_add) - - async def test_add_loop_while_running(self): - self.manager._running = True - task = self.manager.add_loop(self.custom_loop) - self.assertIn(self.custom_loop, self.manager._loops) - self.assertIsInstance(task, asyncio.Task) - - async def test_remove_loop(self): - self.manager._loops.append(self.custom_loop) - self.manager._loop_tasks.append(asyncio.create_task(asyncio.sleep(1))) - self.manager.remove_loop(self.custom_loop) - self.assertNotIn(self.custom_loop, self.manager._loops) - - async def test_run_forever_async(self): - self.manager._running = True - task = asyncio.create_task(self.manager.run_forever_async()) - await asyncio.sleep(0.5) - self.manager._running = False - await asyncio.sleep(0.1) - task.cancel() - - async def test_stop(self): - self.manager._running = True - self.manager.stop() - self.assertFalse(self.manager.running) - self.assertEqual(len(self.manager._loops), 0) - - async def test_run_forever_threaded(self): - self.manager._running = True - import threading - - thread = threading.Thread(target=self.manager.run_forever) - thread.start() - await asyncio.sleep(1) - self.manager.stop() - await asyncio.sleep(0.1) - thread.join() - self.assertFalse(self.manager.running) - - -if __name__ == "__main__": - unittest.main() +@pytest.fixture +def logger(): + return logging.getLogger("TestLogger") + + +@pytest.fixture +def test_loop(logger): + loop = _TestLoop(delay=0.2, logger=logger) + loop.loop = AsyncMock() + return loop + + +@funcnodes_test(no_prefix=True) +async def test_initial_state(test_loop): + assert not test_loop.running + assert not test_loop.stopped + assert test_loop.manager is None + + +@funcnodes_test(no_prefix=True) +async def test_manager_assignment(test_loop): + mock_manager = Mock() + test_loop.manager = mock_manager + assert test_loop.manager == mock_manager + + +@funcnodes_test(no_prefix=True) +async def test_manager_reassignment_fails(test_loop): + mock_manager = Mock() + test_loop.manager = mock_manager + with pytest.raises(ValueError): + test_loop.manager = Mock() + + +@funcnodes_test(no_prefix=True) +async def test_stop_loop(test_loop): + test_loop._running = True + await test_loop.stop() + assert not test_loop.running + assert test_loop.stopped + + +@funcnodes_test(no_prefix=True) +async def test_pause(): + class CountingLoop(CustomLoop): + counter = 0 + + async def loop(self): + self.counter += 1 + + loop = CountingLoop() + asyncio.create_task(loop.continuous_run()) + await asyncio.sleep(0.3) + assert loop.counter > 1 + loop.pause() + fixed_counter = loop.counter + await asyncio.sleep(0.3) + assert loop.counter == fixed_counter + loop.resume() + await asyncio.sleep(0.3) + assert loop.counter > fixed_counter + loop.pause() + fixed_counter = loop.counter + await asyncio.sleep(0.3) + assert loop.counter == fixed_counter + loop.resume_in(1) + await asyncio.sleep(0.5) + assert loop.counter == fixed_counter + await asyncio.sleep(1) + assert loop.counter > fixed_counter + + +@funcnodes_test(no_prefix=True) +async def test_continuous_run_calls_loop(test_loop): + test_loop._running = True + task = asyncio.create_task(test_loop.continuous_run()) + await asyncio.sleep(0.3) + test_loop._running = False + task.cancel() + with suppress(asyncio.CancelledError): + await task + test_loop.loop.assert_called() + + +@pytest.fixture +def loop_manager(): + worker = Mock() + worker.logger = logging.getLogger("TestWorkerLogger") + return LoopManager(worker) + + +@pytest.fixture +def custom_loop(): + return AsyncMock(spec=CustomLoop) + + +@funcnodes_test(no_prefix=True) +async def test_add_loop_while_stopped(loop_manager, custom_loop): + loop_manager.add_loop(custom_loop) + assert custom_loop in loop_manager._loops_to_add + + +@funcnodes_test(no_prefix=True) +async def test_add_loop_while_running(loop_manager, custom_loop): + loop_manager._running = True + task = loop_manager.add_loop(custom_loop) + assert custom_loop in loop_manager._loops + assert isinstance(task, asyncio.Task) + task.cancel() + with suppress(asyncio.CancelledError): + await task + + +@funcnodes_test(no_prefix=True) +async def test_remove_loop(loop_manager, custom_loop): + loop_manager._loops.append(custom_loop) + loop_manager._loop_tasks.append(asyncio.create_task(asyncio.sleep(1))) + loop_manager.remove_loop(custom_loop) + assert custom_loop not in loop_manager._loops + + +@funcnodes_test(no_prefix=True) +async def test_run_forever_async(loop_manager): + loop_manager._running = True + task = asyncio.create_task(loop_manager.run_forever_async()) + await asyncio.sleep(0.5) + loop_manager._running = False + await asyncio.sleep(0.1) + task.cancel() + with suppress(asyncio.CancelledError): + await task + + +@funcnodes_test(no_prefix=True) +async def test_stop_loop_manager(loop_manager): + loop_manager._running = True + loop_manager.stop() + assert not loop_manager.running + assert len(loop_manager._loops) == 0 + + +@funcnodes_test(no_prefix=True) +async def test_run_forever_threaded(loop_manager): + loop_manager._running = True + import threading + + thread = threading.Thread(target=loop_manager.run_forever) + thread.start() + await asyncio.sleep(1) + loop_manager.stop() + await asyncio.sleep(0.1) + thread.join() + assert not loop_manager.running From 45bb7957fad2f9c762389eb6f3d923f4d1c7039c Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Sun, 30 Nov 2025 11:45:42 +0100 Subject: [PATCH 17/19] fix(loop): await task shutdown and prune stale worker state - await cancelled loop tasks and give async_call tasks a timed grace period, even when stop runs inside the manager loop - guard async scheduling with closed/running loop checks to avoid unawaited coroutine warnings - add write/drain helper for socket worker to handle coroutine-returning mocks and real writers - clear stale worker process/runstate files on init and migrate external worker tests to pytest fixtures --- src/funcnodes_worker/loop.py | 131 ++++++- src/funcnodes_worker/socket.py | 24 +- src/funcnodes_worker/worker.py | 15 +- tests/test_external_worker.py | 665 +++++++++++++++------------------ tests/test_loops.py | 27 +- tests/test_socketworker.py | 5 +- 6 files changed, 475 insertions(+), 392 deletions(-) diff --git a/src/funcnodes_worker/loop.py b/src/funcnodes_worker/loop.py index 1150cae..2d9ae09 100644 --- a/src/funcnodes_worker/loop.py +++ b/src/funcnodes_worker/loop.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod import asyncio +import threading from typing import List, Optional import logging import warnings @@ -106,6 +107,7 @@ def __init__(self, worker) -> None: self._loop_tasks: List[asyncio.Task] = [] self._running = False self._loops_to_add = [] + self._async_tasks = [] def reset_loop(self): try: @@ -125,7 +127,7 @@ def reset_loop(self): or "There is no running event loop" in error_msg ): self._loop = asyncio.new_event_loop() - asyncio.set_event_loop(self._loop) + # asyncio.set_event_loop(self._loop) else: raise @@ -167,15 +169,62 @@ def remove_loop(self, loop: CustomLoop): idx = self._loops.index(loop) task = self._loop_tasks.pop(idx) self._loops.pop(idx) - task.cancel() + self._cancel_and_await_task(task, is_running) + + def _cancel_and_await_task(self, task: asyncio.Task, is_running: bool): + """Cancel a task and ensure the cancellation is awaited to release references.""" + + async def _wait_cancel(t: asyncio.Task): + try: + await t + except asyncio.CancelledError: + pass + except Exception as exc: # pragma: no cover - defensive + worker = self._worker() + if worker is not None: + worker.logger.exception(exc) + + if task.done(): + try: + _ = task.exception() + except asyncio.CancelledError: + pass + except Exception as exc: # pragma: no cover - defensive + worker = self._worker() + if worker is not None: + worker.logger.exception(exc) + return + + task.cancel() + + waiter = _wait_cancel(task) + if not is_running and not self._loop.is_closed(): + try: + self._loop.run_until_complete(waiter) + except Exception as exc: # pragma: no cover - defensive + worker = self._worker() + if worker is not None: + worker.logger.exception(exc) + else: + scheduled = self.async_call(waiter) + if scheduled is None: + # no loop to schedule on; close coroutine to avoid "never awaited" + waiter.close() def async_call(self, croutine: asyncio.Coroutine): # Check if the loop is closed or not running + + self._async_tasks: List[asyncio.Task] = [ + t for t in self._async_tasks if not t.done() and not t.cancelled() + ] + if self._loop.is_closed(): # Try to get the running loop instead try: running_loop = asyncio.get_running_loop() - return running_loop.create_task(croutine) + task = running_loop.create_task(croutine) + self._async_tasks.append(task) + return task except RuntimeError: # No running loop available, skip creating the task worker = self._worker() @@ -183,6 +232,8 @@ def async_call(self, croutine: asyncio.Coroutine): worker.logger.warning( "Cannot create task: event loop is closed and no running loop available" ) + # close coroutine to avoid runtime warning about never awaited + croutine.close() return None # Check if the loop is running @@ -190,12 +241,18 @@ def async_call(self, croutine: asyncio.Coroutine): # Try to get the running loop instead try: running_loop = asyncio.get_running_loop() - return running_loop.create_task(croutine) + task = running_loop.create_task(croutine) + self._async_tasks.append(task) + return task except RuntimeError: # No running loop available, but our loop exists, try to use it - pass + # cannot schedule; close coroutine to avoid warnings + croutine.close() + return None - return self._loop.create_task(croutine) + task = self._loop.create_task(croutine) + self._async_tasks.append(task) + return task def __del__(self): self.stop() @@ -205,8 +262,57 @@ def stop(self): for loop in list(self._loops): self.remove_loop(loop) - for task in self._loop_tasks: - task.cancel() + self._async_tasks: List[asyncio.Task] = [ + t for t in self._async_tasks if not t.done() and not t.cancelled() + ] + + is_running = self._loop.is_running() + + # Give pending async_call tasks a brief chance to finish when loop is running + grace_handled = False + if is_running and self._async_tasks: + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + + same_thread = running_loop is self._loop + if not same_thread: + try: + fut = asyncio.run_coroutine_threadsafe( + asyncio.wait(list(self._async_tasks), timeout=1), + self._loop, + ) + fut.result(timeout=2) + grace_handled = True + except Exception as exc: # pragma: no cover - defensive + worker = self._worker() + if worker is not None: + worker.logger.exception(exc) + else: + + async def _grace_wait(tasks: list[asyncio.Task]): + try: + await asyncio.wait(tasks, timeout=1) + except Exception as exc: # pragma: no cover - defensive + worker = self._worker() + if worker is not None: + worker.logger.exception(exc) + finally: + for task in list(tasks): + self._cancel_and_await_task(task, True) + + self._loop.create_task(_grace_wait(list(self._async_tasks))) + grace_handled = True + + # Cancel and await all loop tasks + for task in list(self._loop_tasks): + self._cancel_and_await_task(task, is_running) + + # Cancel and await async_call tasks + if not grace_handled: + for task in list(self._async_tasks): + self._cancel_and_await_task(task, is_running) @property def running(self) -> bool: @@ -225,11 +331,13 @@ def _prerun(self): if worker is not None: worker.logger.info("Starting loop manager") - def run_forever(self): + def run_forever(self, reset_loop: bool = False): try: running_loop = asyncio.get_running_loop() except RuntimeError: running_loop = None + if reset_loop: + self.reset_loop() asyncio.set_event_loop(self._loop) self._prerun() @@ -254,6 +362,11 @@ async def _rf(): if running_loop is not None: asyncio.set_event_loop(running_loop) + def run_forever_threaded(self): + thread = threading.Thread(target=self.run_forever, kwargs={"reset_loop": True}) + thread.start() + return thread + async def run_forever_async(self): self._prerun() diff --git a/src/funcnodes_worker/socket.py b/src/funcnodes_worker/socket.py index bf945c1..1cecee2 100644 --- a/src/funcnodes_worker/socket.py +++ b/src/funcnodes_worker/socket.py @@ -9,6 +9,7 @@ from funcnodes_core import NodeSpace, FUNCNODES_LOGGER from funcnodes_worker import CustomLoop from .remote_worker import RemoteWorker, RemoteWorkerJson +from typing import Any STARTPORT = 9382 ENDPORT = 9482 @@ -166,9 +167,8 @@ async def send_bytes( bytemessage = f"{headerstring}\r\n\r\n".encode() + data + self.DELIMITER - async def _send(writer): - writer.write(bytemessage) - await writer.drain() + async def _send(writer: asyncio.StreamWriter): + await self._write_and_drain(writer, bytemessage) if writer: try: @@ -191,9 +191,8 @@ async def sendmessage( """Sends a message to the frontend.""" bytemessage = msg.encode() + self.DELIMITER - async def _send(writer): - writer.write(bytemessage) - await writer.drain() + async def _send(writer: asyncio.StreamWriter): + await self._write_and_drain(writer, bytemessage) if writer: try: @@ -248,3 +247,16 @@ def exportable_config(self) -> dict: conf.pop("host", None) conf.pop("port", None) return conf + + async def _write_and_drain(self, writer: asyncio.StreamWriter, payload: bytes): + """Write bytes to a writer and await drain; tolerates mocks where write returns a coroutine.""" + res: Any = writer.write(payload) + if asyncio.iscoroutine(res): + await res + elif hasattr(res, "__await__"): + await res + drain_res = writer.drain() + if asyncio.iscoroutine(drain_res): + await drain_res + elif hasattr(drain_res, "__await__"): + await drain_res diff --git a/src/funcnodes_worker/worker.py b/src/funcnodes_worker/worker.py index 9fe7834..3da89ab 100644 --- a/src/funcnodes_worker/worker.py +++ b/src/funcnodes_worker/worker.py @@ -623,9 +623,10 @@ def __init__( else uuid ) self._name = name or None - self._data_path: Path = Path( - data_path if data_path else get_worker_dir(self.uuid()) - ) + self._WORKERS_DIR: Path = get_workers_dir() + self._WORKER_DIR: Path = get_worker_dir(self.uuid()) + + self._data_path: Path = Path(data_path if data_path else self._WORKER_DIR) self.data_path = self._data_path self.logger = fn.get_logger(self.uuid(), propagate=False) if debug: @@ -661,15 +662,15 @@ def venvmanager(self): @property def _process_file(self) -> Path: - return get_workers_dir() / f"worker_{self.uuid()}.p" + return self._WORKERS_DIR / f"worker_{self.uuid()}.p" @property def _runstate_file(self) -> Path: - return get_workers_dir() / f"worker_{self.uuid()}.runstate" + return self._WORKERS_DIR / f"worker_{self.uuid()}.runstate" @property def _config_file(self) -> Path: - return get_workers_dir() / f"worker_{self.uuid()}.json" + return self._WORKERS_DIR / f"worker_{self.uuid()}.json" def _check_process_file(self, hard: bool = False): pf = self._process_file @@ -831,7 +832,7 @@ def write_config(self, opt_conf: Optional[WorkerJson] = None) -> WorkerJson: c["pid"] = os.getpid() # if the data_path is the default data_path, set it to None - if c["data_path"] == get_worker_dir(self.uuid()): + if c["data_path"] == self._WORKER_DIR: c["data_path"] = None cfile = self._config_file if not cfile.parent.exists(): diff --git a/tests/test_external_worker.py b/tests/test_external_worker.py index 9174c3b..8dacab0 100644 --- a/tests/test_external_worker.py +++ b/tests/test_external_worker.py @@ -1,31 +1,18 @@ from typing import Optional -from unittest import IsolatedAsyncioTestCase from weakref import ref import funcnodes_core as fn - - -from pytest_funcnodes import setup as fn_setup, teardown as fn_teardown - -from funcnodes_worker import ( # noqa: E402 - FuncNodesExternalWorker, - RemoteWorker, -) -from unittest.mock import MagicMock # noqa: E402 - - -from funcnodes_core import ( # noqa: E402 - instance_nodefunction, - flatten_shelf, -) -from funcnodes_core.node import register_node # noqa: E402 +from funcnodes_core import instance_nodefunction, flatten_shelf # noqa: E402 from funcnodes_worker import CustomLoop # noqa: E402 +from funcnodes_worker import FuncNodesExternalWorker, RemoteWorker # noqa: E402 +from unittest.mock import MagicMock # noqa: E402 import time # noqa: E402 import asyncio # noqa: E402 import logging # noqa: E402 - import tempfile # noqa: E402 import json # noqa: E402 import gc # noqa: E402 +import pytest +from pytest_funcnodes import funcnodes_test try: import objgraph # noqa: E402 @@ -63,78 +50,6 @@ async def send_bytes(self, *args, **kwargs): return MagicMock() -class TestExternalWorker(IsolatedAsyncioTestCase): - def setUp(self) -> None: - fn_setup() - register_node(workertestnode) - - def tearDown(self) -> None: - fn_teardown() - - def test_external_worker_missing_nodeclassid(self): - with self.assertRaises(ValueError): - - class ExternalWorker2(FuncNodesExternalWorker): - IS_ABSTRACT = False - - async def loop(self): - pass - - async def test_external_worker_sync_loop(self): - class ExternalWorker1(FuncNodesExternalWorker): - NODECLASSID = "testexternalworker" - - def loop(self): - pass - - worker = ExternalWorker1(workerid="test") - worker._logger = RaiseErrorLogger("raiserror") - await asyncio.sleep(0.5) - - with self.assertRaises(TypeError) as e: - await worker.continuous_run() - - self.assertEqual( - "object NoneType can't be used in 'await' expression", str(e.exception) - ) - - async def test_external_worker_loop(self): - class ExternalWorker1(FuncNodesExternalWorker): - NODECLASSID = "testexternalworker" - - async def loop(self): - await self.stop() - - self.assertEqual(ExternalWorker1.running_instances(), []) - worker = ExternalWorker1(workerid="test") - worker._logger = RaiseErrorLogger("raiserror") - await worker.continuous_run() - - async def test_external_worker_serialization(self): - class ExternalWorker1(FuncNodesExternalWorker): - NODECLASSID = "testexternalworker" - - async def loop(self): - await self.stop() - - @instance_nodefunction() - def test(self, a: int) -> int: - return 1 + a - - worker = ExternalWorker1(workerid="test") - ser = json.loads(json.dumps(worker, cls=fn.JSONEncoder)) - self.assertEqual( - ser, - { - "name": "ExternalWorker1(test)", - "nodeclassid": "testexternalworker", - "running": False, - "uuid": "test", - "config": {}, - }, - ) - - class ExternalWorkerSelfStop(FuncNodesExternalWorker): NODECLASSID = "testexternalworker_ExternalWorkerSelfStop" @@ -170,277 +85,340 @@ def get_count(self) -> int: return self.triggercount -class TestExternalWorkerWithWorker(IsolatedAsyncioTestCase): - async def asyncSetUp(self) -> None: - self.tempdir = tempfile.TemporaryDirectory(prefix="funcnodes") - self.retmoteworker = _TestWorker(data_path=self.tempdir.name) - self._loop = asyncio.get_event_loop() - self.runtask = self._loop.create_task(self.retmoteworker.run_forever_async()) - t = time.time() - while not self.retmoteworker.loop_manager.running and time.time() - t < 50: - if self.runtask.done(): - if self.runtask.exception(): - raise self.runtask.exception() - await asyncio.sleep(1) - if not self.retmoteworker.loop_manager.running: - raise Exception("Worker not running") - - async def asyncTearDown(self): - self.retmoteworker.stop() - - async with asyncio.timeout(5): - await self.runtask - - def setUp(self) -> None: - fn_setup() +@fn.NodeDecorator(node_id="workertestnode") +async def workertestnode(a: int) -> int: + return a + 1 - def tearDown(self) -> None: - if not self.runtask.done(): - self.runtask.cancel() - fn_teardown() - self.tempdir.cleanup() - return super().tearDown() +class ExternalWorkerWithNodeShelves(FuncNodesExternalWorker): + NODECLASSID = "testexternalworker_ExternalWorkerWithNodeShelves" - async def test_external_worker_nodes(self): - self.retmoteworker.add_local_worker( - ExternalWorker1, "test_external_worker_nodes" + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._nodeshelf = fn.Shelf( + name="test", + description="test", + nodes=[ + workertestnode, + ], ) - nodeid = "testexternalworker_ExternalWorker1.test_external_worker_nodes.test" - nodeclass = self.retmoteworker.nodespace.lib.get_node_by_id(nodeid) - self.assertEqual(nodeclass.node_name, "Test") - node = self.retmoteworker.add_node(nodeid, name="TestNode") - self.maxDiff = None - expected_node_ser = { - "name": "TestNode", - "id": node.uuid, - "node_id": nodeid, - "node_name": "Test", - "io": { - "a": {"is_input": True, "value": fn.NoValue, "emit_value_set": True}, - "out": {"is_input": False, "value": fn.NoValue, "emit_value_set": True}, - }, - } - self.assertEqual(node.serialize(), expected_node_ser) - - async def test_base_run(self): - for _ in range(5): - await asyncio.sleep(0.3) - t = time.time() - self.assertLessEqual(t - self.retmoteworker.timerloop.last_run, 0.25) - async def test_external_worker_run(self): - def get_ws_nodes(): - nodes = [] - for shelf in self.retmoteworker.nodespace.lib.shelves: - nodes.extend(flatten_shelf(shelf)[0]) - return nodes - - def check_nodes_length(target=0): - nodes = get_ws_nodes() - - if target == 0 and len(nodes) > 0 and objgraph: - objgraph.show_backrefs( - nodes, - max_depth=15, - filename="backrefs_nodes.dot", - highlight=lambda x: isinstance(x, fn.Node), - shortnames=False, - ) - - self.assertEqual(len(nodes), target, nodes) + def get_nodeshelf(self) -> Optional[fn.Shelf]: + return self._nodeshelf - del nodes - gc.collect() - await asyncio.sleep(0.5) - t = time.time() - self.assertLessEqual( - t - self.retmoteworker.timerloop.last_run, - 0.4, - (t, self.retmoteworker.timerloop.last_run), - ) - print("adding worker") - check_nodes_length(0) +# @pytest.fixture(autouse=True) +# def funcnodes_setup_teardown(): +# fn_setup() +# register_node(workertestnode) +# yield +# fn_teardown() + + +@pytest.fixture +async def running_remote_worker(): + tempdir = tempfile.TemporaryDirectory(prefix="funcnodes") + retmoteworker = _TestWorker(data_path=tempdir.name) + loop = asyncio.get_event_loop() + runtask = loop.create_task(retmoteworker.run_forever_async()) + start = time.time() + while not retmoteworker.loop_manager.running and time.time() - start < 50: + if runtask.done(): + if runtask.exception(): + tempdir.cleanup() + raise runtask.exception() + await asyncio.sleep(1) + if not retmoteworker.loop_manager.running: + runtask.cancel() + tempdir.cleanup() + raise Exception("Worker not running") + try: + yield retmoteworker + finally: + retmoteworker.stop() + async with asyncio.timeout(5): + await runtask + tempdir.cleanup() - w: ExternalWorker1 = self.retmoteworker.add_local_worker( - ExternalWorker1, "test_external_worker_run" - ) - check_nodes_length(2) +@funcnodes_test(no_prefix=True) +def test_external_worker_missing_nodeclassid(): + with pytest.raises(ValueError): - self.assertIn( - "testexternalworker_ExternalWorker1", - FuncNodesExternalWorker.RUNNING_WORKERS, - ) - self.assertIn( - "test_external_worker_run", - FuncNodesExternalWorker.RUNNING_WORKERS[ - "testexternalworker_ExternalWorker1" - ], - ) + class ExternalWorker2(FuncNodesExternalWorker): + IS_ABSTRACT = False - nodetest = self.retmoteworker.add_node( - "testexternalworker_ExternalWorker1.test_external_worker_run.test", - ) + async def loop(self): + pass - node_getcount = self.retmoteworker.add_node( - "testexternalworker_ExternalWorker1.test_external_worker_run.get_count", - ) - self.assertIn("out", node_getcount.outputs, node_getcount.outputs.keys()) - self.assertEqual(node_getcount.outputs["out"].value, fn.NoValue) - self.assertEqual(w.triggercount, 0) +@funcnodes_test(no_prefix=True) +async def test_external_worker_sync_loop(): + class ExternalWorker1(FuncNodesExternalWorker): + NODECLASSID = "testexternalworker" - fn.FUNCNODES_LOGGER.debug("triggering node_getcount 1") - await node_getcount + def loop(self): + pass - self.assertEqual(node_getcount.outputs["out"].value, 0) - self.assertEqual(w.triggercount, 0) + assert ExternalWorker1.running_instances() == [], ( + ExternalWorker1.running_instances() + ) + worker = ExternalWorker1(workerid="test") + worker._logger = RaiseErrorLogger("raiserror") + await asyncio.sleep(0.5) - self.assertEqual(w.triggercount, 0) - fn.FUNCNODES_LOGGER.debug("triggering nodetest 1") - nodetest.inputs["a"].value = 1 - await fn.run_until_complete(nodetest) + with pytest.raises(TypeError) as e: + await worker.continuous_run() - self.assertEqual(w.triggercount, 1) - self.assertEqual(nodetest.outputs["out"].value, 2) - fn.FUNCNODES_LOGGER.debug("triggering node_getcount 2") - await node_getcount + assert "object NoneType can't be used in 'await' expression" == str(e.value) + assert worker.running + await worker.stop() + assert not worker.running + assert ExternalWorker1.running_instances() == [], ( + ExternalWorker1.running_instances() + ) + + +@funcnodes_test(no_prefix=True) +async def test_external_worker_loop(): + class ExternalWorker1(FuncNodesExternalWorker): + NODECLASSID = "testexternalworker" + + async def loop(self): + await self.stop() + + assert ExternalWorker1.running_instances() == [], ( + ExternalWorker1.running_instances() + ) + worker = ExternalWorker1(workerid="test") + worker._logger = RaiseErrorLogger("raiserror") + await worker.continuous_run() + + +@funcnodes_test(no_prefix=True) +async def test_external_worker_serialization(): + class ExternalWorker1(FuncNodesExternalWorker): + NODECLASSID = "testexternalworker" + + async def loop(self): + await self.stop() + + @instance_nodefunction() + def test(self, a: int) -> int: + return 1 + a + + worker = ExternalWorker1(workerid="test") + ser = json.loads(json.dumps(worker, cls=fn.JSONEncoder)) + assert ser == { + "name": "ExternalWorker1(test)", + "nodeclassid": "testexternalworker", + "running": False, + "uuid": "test", + "config": {}, + } + + +@funcnodes_test(no_prefix=True) +async def test_external_worker_nodes(running_remote_worker: _TestWorker): + running_remote_worker.add_local_worker( + ExternalWorker1, "test_external_worker_nodes" + ) + nodeid = "testexternalworker_ExternalWorker1.test_external_worker_nodes.test" + nodeclass = running_remote_worker.nodespace.lib.get_node_by_id(nodeid) + assert nodeclass.node_name == "Test" + node = running_remote_worker.add_node(nodeid, name="TestNode") + expected_node_ser = { + "name": "TestNode", + "id": node.uuid, + "node_id": nodeid, + "node_name": "Test", + "io": { + "a": {"is_input": True, "value": fn.NoValue, "emit_value_set": True}, + "out": {"is_input": False, "value": fn.NoValue, "emit_value_set": True}, + }, + } + assert node.serialize() == expected_node_ser + + +@funcnodes_test(no_prefix=True) +async def test_base_run(running_remote_worker: _TestWorker): + for _ in range(5): + await asyncio.sleep(0.3) + t = time.time() + assert t - running_remote_worker.timerloop.last_run <= 0.25 + + +@funcnodes_test(no_prefix=True) +async def test_external_worker_run(running_remote_worker: _TestWorker): + def get_ws_nodes(): + nodes = [] + for shelf in running_remote_worker.nodespace.lib.shelves: + nodes.extend(flatten_shelf(shelf)[0]) + return nodes + + def check_nodes_length(target=0): + nodes = get_ws_nodes() + + if target == 0 and len(nodes) > 0 and objgraph: + objgraph.show_backrefs( + nodes, + max_depth=15, + filename="backrefs_nodes.dot", + highlight=lambda x: isinstance(x, fn.Node), + shortnames=False, + ) - self.assertIn("out", node_getcount.outputs, node_getcount.outputs.keys()) - self.assertEqual(node_getcount.outputs["out"].value, 1) + assert len(nodes) == target, nodes - self.assertEqual( - nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"], - False, - ) + del nodes + gc.collect() - w.increment_trigger() - self.assertEqual( - nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"], - True, - ) + await asyncio.sleep(0.5) + t = time.time() + assert t - running_remote_worker.timerloop.last_run <= 0.4 + print("adding worker") + check_nodes_length(0) + + w: ExternalWorker1 = running_remote_worker.add_local_worker( + ExternalWorker1, "test_external_worker_run" + ) + + check_nodes_length(2) + + assert ( + "testexternalworker_ExternalWorker1" in FuncNodesExternalWorker.RUNNING_WORKERS + ) + assert ( + "test_external_worker_run" + in FuncNodesExternalWorker.RUNNING_WORKERS["testexternalworker_ExternalWorker1"] + ) + + nodetest = running_remote_worker.add_node( + "testexternalworker_ExternalWorker1.test_external_worker_run.test", + ) + + node_getcount = running_remote_worker.add_node( + "testexternalworker_ExternalWorker1.test_external_worker_run.get_count", + ) + + assert "out" in node_getcount.outputs + assert node_getcount.outputs["out"].value is fn.NoValue + assert w.triggercount == 0 + + fn.FUNCNODES_LOGGER.debug("triggering node_getcount 1") + await node_getcount + + assert node_getcount.outputs["out"].value == 0 + assert w.triggercount == 0 + + fn.FUNCNODES_LOGGER.debug("triggering nodetest 1") + nodetest.inputs["a"].value = 1 + await fn.run_until_complete(nodetest) + + assert w.triggercount == 1 + assert nodetest.outputs["out"].value == 2 + fn.FUNCNODES_LOGGER.debug("triggering node_getcount 2") + await node_getcount + + assert "out" in node_getcount.outputs + assert node_getcount.outputs["out"].value == 1 + + assert not ( + nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"] + ) + + w.increment_trigger() + assert nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"] + await asyncio.sleep(0.1) + print("waiting") + t = time.time() + while ( + nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"] + ) and time.time() - t < 10: await asyncio.sleep(0.1) - print("waiting") - t = time.time() - while ( - nodetest.status()["requests_trigger"] or nodetest.status()["in_trigger"] - ) and time.time() - t < 10: - await asyncio.sleep(0.1) - t = time.time() - while not w.stopped and time.time() - t < 10: - print(w._stopped, w._running) - await asyncio.sleep(0.6) - await w.stop() - del w - del node_getcount - del nodetest - await asyncio.sleep(5) - - # await asyncio.sleep(6) - t = time.time() - self.assertLessEqual(t - self.retmoteworker.timerloop.last_run, 1.0) - gc.collect() + t = time.time() + while not w.stopped and time.time() - t < 10: + print(w._stopped, w._running) + await asyncio.sleep(0.6) + await w.stop() + del w + del node_getcount + del nodetest + await asyncio.sleep(5) + + t = time.time() + assert t - running_remote_worker.timerloop.last_run <= 1.0 + gc.collect() + if "testexternalworker_ExternalWorker1" in FuncNodesExternalWorker.RUNNING_WORKERS: if ( - "testexternalworker_ExternalWorker1" - in FuncNodesExternalWorker.RUNNING_WORKERS + "test_external_worker_run" + in FuncNodesExternalWorker.RUNNING_WORKERS[ + "testexternalworker_ExternalWorker1" + ] ): - if ( - "test_external_worker_run" - in FuncNodesExternalWorker.RUNNING_WORKERS[ - "testexternalworker_ExternalWorker1" - ] - ): - if objgraph: - objgraph.show_backrefs( - [ - FuncNodesExternalWorker.RUNNING_WORKERS[ - "testexternalworker_ExternalWorker1" - ]["test_external_worker_run"] - ], - max_depth=10, - filename="backrefs_before.dot", - highlight=lambda x: isinstance(x, ExternalWorker1), - shortnames=False, - ) - - self.assertNotIn( - "test_external_worker_run", - FuncNodesExternalWorker.RUNNING_WORKERS[ - "testexternalworker_ExternalWorker1" - ], - ) - - check_nodes_length(0) - - await asyncio.sleep(0.5) - t = time.time() - self.assertLessEqual(t - self.retmoteworker.timerloop.last_run, 0.3) - - -@fn.NodeDecorator(node_id="workertestnode") -async def workertestnode(a: int) -> int: - return a + 1 - - -class ExternalWorkerWithNodeShelves(FuncNodesExternalWorker): - NODECLASSID = "testexternalworker_ExternalWorkerWithNodeShelves" + if objgraph: + objgraph.show_backrefs( + [ + FuncNodesExternalWorker.RUNNING_WORKERS[ + "testexternalworker_ExternalWorker1" + ]["test_external_worker_run"] + ], + max_depth=10, + filename="backrefs_before.dot", + highlight=lambda x: isinstance(x, ExternalWorker1), + shortnames=False, + extra_node_attrs=lambda x: {"longname": str(x)}, + ) - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self._nodeshelf = fn.Shelf( - name="test", - description="test", - nodes=[ - workertestnode, - ], - ) + assert ( + "test_external_worker_run" + not in FuncNodesExternalWorker.RUNNING_WORKERS[ + "testexternalworker_ExternalWorker1" + ] + ), { + k: {vk: vv for vk, vv in v.items()} + for k, v in FuncNodesExternalWorker.RUNNING_WORKERS.items() + } - def get_nodeshelf(self) -> Optional[fn.Shelf]: - return self._nodeshelf + check_nodes_length(0) + await asyncio.sleep(0.5) + t = time.time() + assert t - running_remote_worker.timerloop.last_run <= 0.3 -class TestExternalWorkerWithNodeShelves(IsolatedAsyncioTestCase): - async def asyncSetUp(self) -> None: - self.tempdir = tempfile.TemporaryDirectory(prefix="funcnodes") - self.retmoteworker = _TestWorker(data_path=self.tempdir.name) - self._loop = asyncio.get_event_loop() - self.runtask = self._loop.create_task(self.retmoteworker.run_forever_async()) - t = time.time() - while not self.retmoteworker.loop_manager.running and time.time() - t < 50: - if self.runtask.done(): - if self.runtask.exception(): - raise self.runtask.exception() - await asyncio.sleep(1) - if not self.retmoteworker.loop_manager.running: - raise Exception("Worker not running") - async def asyncTearDown(self): - self.retmoteworker.stop() +@funcnodes_test(no_prefix=True) +async def test_external_worker_nodes_shelf(running_remote_worker: _TestWorker): + worker = running_remote_worker.add_local_worker( + ExternalWorkerWithNodeShelves, "test_external_worker_nodes" + ) - async with asyncio.timeout(5): - await self.runtask + assert isinstance(worker, ExternalWorkerWithNodeShelves) - def setUp(self) -> None: - fn_setup() - # register_node(workertestnode) + assert worker.get_nodeshelf() is not None + assert worker.get_nodeshelf().name == "test" + assert worker.get_nodeshelf().nodes == [workertestnode] - def tearDown(self) -> None: - if not self.runtask.done(): - self.runtask.cancel() + assert isinstance(worker.nodeshelf, ref) + assert worker.nodeshelf() is not None + assert worker.nodeshelf().name == "test" + assert worker.nodeshelf().nodes == [workertestnode] - fn_teardown() - self.tempdir.cleanup() - return super().tearDown() + nodeclass = running_remote_worker.nodespace.lib.get_node_by_id("workertestnode") + assert nodeclass.node_name == "workertestnode" - async def test_external_worker_nodes(self): - worker = self.retmoteworker.add_local_worker( - ExternalWorkerWithNodeShelves, "test_external_worker_nodes" - ) - assert isinstance(worker, ExternalWorkerWithNodeShelves) +@funcnodes_test(no_prefix=True) +async def test_external_worker_nodes_multiple_updates( + running_remote_worker: _TestWorker, +): + worker = running_remote_worker.add_local_worker( + ExternalWorkerWithNodeShelves, "test_external_worker_nodes_multiple" + ) + for _ in range(2): + print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) + worker.emit("nodes_update") assert worker.get_nodeshelf() is not None assert worker.get_nodeshelf().name == "test" assert worker.get_nodeshelf().nodes == [workertestnode] @@ -450,35 +428,10 @@ async def test_external_worker_nodes(self): assert worker.nodeshelf().name == "test" assert worker.nodeshelf().nodes == [workertestnode] - nodeclass = self.retmoteworker.nodespace.lib.get_node_by_id("workertestnode") - self.assertEqual(nodeclass.node_name, "workertestnode") - - async def test_external_worker_nodes_multiple_updates(self): - worker = self.retmoteworker.add_local_worker( - ExternalWorkerWithNodeShelves, "test_external_worker_nodes_multiple" + print( + json.dumps(running_remote_worker.nodespace.lib.full_serialize(), indent=4) ) + print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) - for _ in range(2): - print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) - worker.emit("nodes_update") - assert worker.get_nodeshelf() is not None - assert worker.get_nodeshelf().name == "test" - assert worker.get_nodeshelf().nodes == [workertestnode] - - assert isinstance(worker.nodeshelf, ref) - assert worker.nodeshelf() is not None - assert worker.nodeshelf().name == "test" - assert worker.nodeshelf().nodes == [workertestnode] - - print( - json.dumps(self.retmoteworker.nodespace.lib.full_serialize(), indent=4) - ) - print("registered nodes", list(fn.node.REGISTERED_NODES.keys())) - - nodeclass = self.retmoteworker.nodespace.lib.get_node_by_id( - "workertestnode" - ) - self.assertEqual(nodeclass.node_name, "workertestnode") - - -# + nodeclass = running_remote_worker.nodespace.lib.get_node_by_id("workertestnode") + assert nodeclass.node_name == "workertestnode" diff --git a/tests/test_loops.py b/tests/test_loops.py index 8ab12a6..8051d16 100644 --- a/tests/test_loops.py +++ b/tests/test_loops.py @@ -1,6 +1,7 @@ import asyncio import logging from contextlib import suppress +import time from unittest.mock import AsyncMock, Mock import pytest @@ -117,13 +118,17 @@ def custom_loop(): @funcnodes_test(no_prefix=True) -async def test_add_loop_while_stopped(loop_manager, custom_loop): +async def test_add_loop_while_stopped( + loop_manager: LoopManager, custom_loop: CustomLoop +): loop_manager.add_loop(custom_loop) assert custom_loop in loop_manager._loops_to_add @funcnodes_test(no_prefix=True) -async def test_add_loop_while_running(loop_manager, custom_loop): +async def test_add_loop_while_running( + loop_manager: LoopManager, custom_loop: CustomLoop +): loop_manager._running = True task = loop_manager.add_loop(custom_loop) assert custom_loop in loop_manager._loops @@ -134,7 +139,7 @@ async def test_add_loop_while_running(loop_manager, custom_loop): @funcnodes_test(no_prefix=True) -async def test_remove_loop(loop_manager, custom_loop): +async def test_remove_loop(loop_manager: LoopManager, custom_loop: CustomLoop): loop_manager._loops.append(custom_loop) loop_manager._loop_tasks.append(asyncio.create_task(asyncio.sleep(1))) loop_manager.remove_loop(custom_loop) @@ -142,7 +147,7 @@ async def test_remove_loop(loop_manager, custom_loop): @funcnodes_test(no_prefix=True) -async def test_run_forever_async(loop_manager): +async def test_run_forever_async(loop_manager: LoopManager): loop_manager._running = True task = asyncio.create_task(loop_manager.run_forever_async()) await asyncio.sleep(0.5) @@ -154,7 +159,7 @@ async def test_run_forever_async(loop_manager): @funcnodes_test(no_prefix=True) -async def test_stop_loop_manager(loop_manager): +async def test_stop_loop_manager(loop_manager: LoopManager): loop_manager._running = True loop_manager.stop() assert not loop_manager.running @@ -162,13 +167,11 @@ async def test_stop_loop_manager(loop_manager): @funcnodes_test(no_prefix=True) -async def test_run_forever_threaded(loop_manager): - loop_manager._running = True - import threading - - thread = threading.Thread(target=loop_manager.run_forever) - thread.start() - await asyncio.sleep(1) +async def test_run_forever_threaded(loop_manager: LoopManager): + thread = loop_manager.run_forever_threaded() + start = time.time() + while not loop_manager.running and time.time() - start < 10: + await asyncio.sleep(0.1) loop_manager.stop() await asyncio.sleep(0.1) thread.join() diff --git a/tests/test_socketworker.py b/tests/test_socketworker.py index 9ab6166..9ba5144 100644 --- a/tests/test_socketworker.py +++ b/tests/test_socketworker.py @@ -44,10 +44,11 @@ async def test_send_message_to_clients(worker): @funcnodes_test(no_prefix=True) -async def test_stop(worker): +async def test_stop(worker: SocketWorker): asyncio.create_task(worker.run_forever_async()) - await worker.wait_for_running(timeout=10) + await worker.wait_for_running(timeout=20) await asyncio.sleep(1) assert worker.socket_loop.running worker.stop() + await asyncio.sleep(2) assert not worker.socket_loop.running From 1f7f0779c3c49a4bece220597c4bfaf2cc2f3aa8 Mon Sep 17 00:00:00 2001 From: JulianKimmig Date: Fri, 5 Dec 2025 10:06:01 +0100 Subject: [PATCH 18/19] feat(worker): add optional name and config parameters to external worker initialization - Updated FuncNodesExternalWorker to accept an optional name parameter during initialization. - Modified LocalWorkerLookupLoop and Worker classes to pass the new name and config parameters when starting local workers. - Enhanced add_local_worker method to support additional configuration options for external workers. --- src/funcnodes_worker/external_worker.py | 3 +++ src/funcnodes_worker/worker.py | 33 ++++++++++++++++++------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/funcnodes_worker/external_worker.py b/src/funcnodes_worker/external_worker.py index 128d1f6..2cec5ab 100644 --- a/src/funcnodes_worker/external_worker.py +++ b/src/funcnodes_worker/external_worker.py @@ -57,6 +57,7 @@ def __init__( workerid, config: Optional[Union[ExternalWorkerConfig, Dict[str, Any]]] = None, data_path: Optional[str] = None, + name: Optional[str] = None, ) -> None: """ Initializes the FuncNodesExternalWorker class. @@ -71,6 +72,8 @@ def __init__( self._nodeshelf: Optional[Shelf] = None self._config = self.config_cls() self._data_path: Optional[Path] = Path(data_path) if data_path else None + if name: + self.name = name try: self.update_config(config) except Exception: diff --git a/src/funcnodes_worker/worker.py b/src/funcnodes_worker/worker.py index 3da89ab..c704369 100644 --- a/src/funcnodes_worker/worker.py +++ b/src/funcnodes_worker/worker.py @@ -271,7 +271,11 @@ def _worker_instance_stopping_callback( self._client.logger.exception(e) def start_local_worker( - self, worker_class: Type[FuncNodesExternalWorker], worker_id: str + self, + worker_class: Type[FuncNodesExternalWorker], + worker_id: str, + name: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, ): if worker_class not in self.worker_classes: self.worker_classes.append(worker_class) @@ -282,6 +286,8 @@ def start_local_worker( worker_instance: FuncNodesExternalWorker = worker_class( workerid=worker_id, data_path=self._client.data_path / "external_workers" / worker_id, + name=name, + config=config, ) worker_instance.on( @@ -1055,8 +1061,16 @@ def nodespace_id(self) -> str: # endregion properties # region local worker - def add_local_worker(self, worker_class: Type[FuncNodesExternalWorker], nid: str): - w = self.local_worker_lookup_loop.start_local_worker(worker_class, nid) + def add_local_worker( + self, + worker_class: Type[FuncNodesExternalWorker], + nid: str, + name: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + ): + w = self.local_worker_lookup_loop.start_local_worker( + worker_class, nid, name=name, config=config + ) self.loop_manager.async_call(self.worker_event("external_worker_update")) return w @@ -1357,13 +1371,14 @@ async def load(self, data: WorkerState | str | None = None): if worker.NODECLASSID == worker_id: for instance in worker_uuid: if isinstance(instance, str): - w = self.add_local_worker(worker, instance) + self.add_local_worker(worker, instance) else: - w = self.add_local_worker(worker, instance["uuid"]) - if "name" in instance: - w.name = instance["name"] - if "config" in instance: - w.update_config(instance["config"]) + self.add_local_worker( + worker, + nid=instance["uuid"], + name=instance.get("name", None), + config=instance.get("config", None), + ) found = True if not found: self.logger.warning(f"External worker {worker_id} not found") From 973c5e0e3ac5b1ac42c34db668a4f8df362427d9 Mon Sep 17 00:00:00 2001 From: Julian Kimmig Date: Fri, 5 Dec 2025 10:10:58 +0100 Subject: [PATCH 19/19] =?UTF-8?q?bump:=20version=201.3.0=20=E2=86=92=201.4?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 19 +++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41f678c..de0d1a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 1.4.0 (2025-12-05) + +### Feat + +- **worker**: add optional name and config parameters to external worker initialization + +### Fix + +- **loop**: await task shutdown and prune stale worker state + +### Refactor + +- **tests**: migrate test_loops to pytest and implement fixtures +- **tests**: migrate WSWorker tests to pytest and implement fixtures +- **tests**: remove unused pytestmark from test_socketworker.py +- **tests**: migrate SocketWorker tests to pytest and utilize fixtures +- **tests**: migrate test_client_connection to pytest and improve test structure +- **tests**: update worker fixture to use test name for UUID + ## 1.3.0 (2025-11-27) ### Feat diff --git a/pyproject.toml b/pyproject.toml index b5bd0f1..897945f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "funcnodes-worker" -version = "1.3.0" +version = "1.4.0" description = "Worker package for FuncNodes" readme = "README.md" authors = [{name = "Julian Kimmig", email = "julian.kimmig@linkdlab.de"}] diff --git a/uv.lock b/uv.lock index 98192c5..0ee3dcd 100644 --- a/uv.lock +++ b/uv.lock @@ -474,7 +474,7 @@ wheels = [ [[package]] name = "funcnodes-worker" -version = "1.3.0" +version = "1.4.0" source = { editable = "." } dependencies = [ { name = "asynctoolkit" },