[PROPOSAL] Plugin Discovery and Namespace Code Structure

flowsint-api/app/api/routes/events.py

@@ -17,7 +17,8 @@
 from app.api.deps import get_current_user, get_current_user_sse
 
 router = APIRouter()
-
+# do avoid flowsint_enrichers import requires REDIS_URL
+event_emitter.connect()
 
 @router.get("/sketch/{sketch_id}/logs")
 def get_logs_by_sketch(

flowsint-core/src/flowsint_core/core/events.py

@@ -11,9 +11,16 @@
 class EventEmitter:
     def __init__(self):
         self.id = uuid.uuid4()
-        self.redis = redis.from_url(os.environ["REDIS_URL"])
+        self.redis = None
         self.pubsubs: Dict[str, redis.client.PubSub] = {}
 
+    def connect(self):
+        """Connect to Redis server using REDIS_URL environment variable
+        To avoid to connect to Redis when the module is imported (e.g. when importing flowsint_enrichers)
+        """
+        if self.redis is None:
+            self.redis = redis.from_url(os.environ["REDIS_URL"])
+
     async def subscribe(self, channel: str):
         """Subscribe to Redis channel"""
         if channel not in self.pubsubs:

python/.python-version

@@ -0,0 +1 @@
+3.13

python/packages/flowsint-core/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "flowsint-core-next"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+
+]
+
+[build-system]
+requires = ["uv_build >= 0.10.7, <0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "flowsint.core"

python/packages/flowsint-core/src/flowsint/core/__init__.py

@@ -0,0 +1 @@
+from .abc import *

python/packages/flowsint-core/src/flowsint/core/__main__.py

@@ -0,0 +1 @@
+print('root flowsint.core __main__.py')

python/packages/flowsint-enrichers/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "flowsint-enrichers-next"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+  "flowsint-core-next",
+  "flowsint-enrichers",
+]
+
+[tool.uv.sources]
+flowsint-enrichers = { path = "../../../flowsint-enrichers", editable = true }
+flowsint-core-next = { workspace = true, editable = true }
+
+[build-system]
+requires = ["uv_build >= 0.10.7, <0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "flowsint.enricher"

python/packages/flowsint-enrichers/src/flowsint/enricher/__init__.py

@@ -0,0 +1,17 @@
+import importlib.metadata as _importlib_metadata
+
+from flowsint_enrichers.registry import ENRICHER_REGISTRY as _FI_ENRICHER_REGISTRY
+from flowsint_enrichers.domain import *
+
+from .util import importlib as _fi_util_importlib
+
+
+entry_points = _importlib_metadata.entry_points(group='flowsint.enricher')
+plugins = {ep.value: _fi_util_importlib.lazy_load(ep) for ep in entry_points}
+print(plugins)
+for entry_point in entry_points:
+    print(entry_point)
+    print(entry_point.load())
+
+
+print(_FI_ENRICHER_REGISTRY._enrichers)

python/packages/flowsint-enrichers/src/flowsint/enricher/abc.py

@@ -0,0 +1,8 @@
+from flowsint_core.core.enricher_base import Enricher as _EnricherABC
+from flowsint_enrichers.registry import ENRICHER_REGISTRY as _FI_ENRICHER_REGISTRY
+
+
+class EnricherABC(_EnricherABC):
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        _FI_ENRICHER_REGISTRY.register(cls)

python/packages/flowsint-enrichers/src/flowsint/enricher/util/importlib.py

@@ -0,0 +1,185 @@
+"""PEP 810-style explicit lazy imports for Python 3.12+.
+
+Defers module loading until the imported name is first accessed.
+
+Usage::
+
+    import flowsint.util.importlib as _fi_util_importlib
+
+    # Context manager style (PEP 810 syntax)
+    with _fi_util_importlib.lazy():
+        import heavy_module
+        from package import submodule  # works when submodule is a module
+
+    heavy_module.some_func()  # loads here
+
+    # One-liner style
+    heavy_module = _fi_util_importlib.lazy_import("heavy_module")
+    heavy_module.some_func()  # loads here
+
+    # Entry point style — defers ep.load() until first attribute access or call
+    entry_points = importlib.metadata.entry_points(group='myapp.plugins')
+    plugins = [_fi_util_importlib.lazy_load(ep) for ep in entry_points]
+    # nothing loaded yet
+    plugins[0].run()  # loads and calls here
+
+Note:
+    ``from x import name`` where ``name`` is a plain attribute (class, function,
+    constant) will trigger loading of ``x`` at import time because Python must
+    resolve the attribute immediately.  True attribute-level laziness would
+    require transparent proxies for arbitrary Python objects.  This
+    implementation covers the practical and common case: lazy *module* loading.
+
+    ``lazy_load()`` wraps ``EntryPoint.load()`` entirely, so it handles both
+    module-level and attribute-level entry points uniformly.
+"""
+
+from __future__ import annotations
+
+import importlib.abc
+import importlib.metadata
+import importlib.util
+import sys
+import types
+from contextlib import contextmanager
+from typing import Generator
+
+
+class _LazyFinder(importlib.abc.MetaPathFinder):
+    """Meta path hook that wraps resolved loaders with ``importlib.util.LazyLoader``."""
+
+    def find_spec(
+        self,
+        fullname: str,
+        path: object,
+        target: object = None,
+    ) -> importlib.machinery.ModuleSpec | None:
+        # Remove self temporarily to avoid infinite recursion when we call
+        # find_spec recursively through the remaining finders.
+        sys.meta_path.remove(self)
+        try:
+            spec = importlib.util.find_spec(fullname)
+        finally:
+            sys.meta_path.insert(0, self)
+
+        if spec is None or spec.loader is None:
+            return None
+
+        # Module already loaded — let the normal machinery handle it.
+        if fullname in sys.modules:
+            return None
+
+        spec.loader = importlib.util.LazyLoader(spec.loader)
+        return spec
+
+
+@contextmanager
+def lazy() -> Generator[None, None, None]:
+    """Context manager that makes ``import`` statements inside the block lazy.
+
+    Modules imported inside the ``with`` block are not executed until an
+    attribute is accessed on them for the first time.
+
+    Example::
+
+        with lazy():
+            import numpy as np      # not loaded yet
+            import pandas as pd     # not loaded yet
+
+        df = pd.DataFrame()         # pandas loads here
+        arr = np.array([1, 2, 3])   # numpy loads here
+    """
+    finder = _LazyFinder()
+    sys.meta_path.insert(0, finder)
+    try:
+        yield
+    finally:
+        try:
+            sys.meta_path.remove(finder)
+        except ValueError:
+            pass  # already removed (e.g. exception during import lookup)
+
+
+def lazy_import(name: str) -> types.ModuleType:
+    """Return a lazy proxy for *name*; the module loads on first attribute access.
+
+    Unlike ``with lazy(): import name``, this helper works as a single
+    expression and is useful for conditional or programmatic lazy imports.
+
+    Example::
+
+        np = lazy_import("numpy")
+        # numpy is NOT loaded yet
+        arr = np.array([1, 2, 3])   # numpy loads here
+
+    Raises:
+        ModuleNotFoundError: if *name* cannot be found by the import system.
+    """
+    spec = importlib.util.find_spec(name)
+    if spec is None:
+        raise ModuleNotFoundError(f"No module named {name!r}")
+    if spec.loader is None:
+        raise ImportError(f"Module {name!r} has no loader")
+
+    loader = importlib.util.LazyLoader(spec.loader)
+    spec.loader = loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    loader.exec_module(module)
+    return module
+
+
+class _LazyEntryPointProxy:
+    """Transparent proxy that defers ``EntryPoint.load()`` until first use.
+
+    ``EntryPoint.load()`` calls ``importlib.import_module()`` then immediately
+    resolves an attribute path via ``getattr()``, which triggers full module
+    execution even when wrapped in ``with lazy():``.  This proxy defers the
+    entire ``.load()`` call — and therefore both the import and attribute
+    resolution — until the proxy is first accessed or called.
+    """
+
+    __slots__ = ("_ep", "_obj", "_loaded")
+
+    def __init__(self, ep: importlib.metadata.EntryPoint) -> None:
+        object.__setattr__(self, "_ep", ep)
+        object.__setattr__(self, "_obj", None)
+        object.__setattr__(self, "_loaded", False)
+
+    def _resolve(self) -> object:
+        if not object.__getattribute__(self, "_loaded"):
+            ep = object.__getattribute__(self, "_ep")
+            obj = ep.load()
+            object.__setattr__(self, "_obj", obj)
+            object.__setattr__(self, "_loaded", True)
+        return object.__getattribute__(self, "_obj")
+
+    def __getattr__(self, name: str) -> object:
+        return getattr(self._resolve(), name)
+
+    def __call__(self, *args: object, **kwargs: object) -> object:
+        return self._resolve()(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        if object.__getattribute__(self, "_loaded"):
+            return repr(object.__getattribute__(self, "_obj"))
+        ep = object.__getattribute__(self, "_ep")
+        return f"<lazy entry point {ep.value!r}>"
+
+
+def lazy_load(ep: importlib.metadata.EntryPoint) -> _LazyEntryPointProxy:
+    """Return a lazy proxy for an entry point; ``.load()`` is called on first use.
+
+    Unlike wrapping ``ep.load()`` in ``with lazy():``, this defers both the
+    module import *and* the attribute resolution step, so it works correctly
+    for attribute-level entry points (e.g. ``'pkg.module:ClassName'``).
+
+    Example::
+
+        entry_points = importlib.metadata.entry_points(group='flowsint.enricher')
+        enrichers = [lazy_load(ep) for ep in entry_points]
+        # nothing loaded yet
+
+        enrichers[0].scan(data)   # loads the first enricher here
+    """
+    return _LazyEntryPointProxy(ep)

python/packages/flowsint-example-plugin/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "flowsint-example-plugin"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "flowsint-core-next",
+]
+
+[project.entry-points.'flowsint.enricher']
+'flowsint.example.enricher.mock:Enricher' = 'flowsint.example.enricher.mock:Enricher'
+
+[build-system]
+requires = ["uv_build >= 0.10.7, <0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "flowsint.example"
+
+[tool.uv.sources]
+flowsint-core-next = { workspace = true, editable = true }

python/packages/flowsint-example-plugin/src/flowsint/example/enricher/__init__.py

@@ -0,0 +1,2 @@
+
+from .mock import *

python/packages/flowsint-example-plugin/src/flowsint/example/enricher/mock.py

@@ -0,0 +1,38 @@
+import typing as _t
+
+import flowsint.types as _fi_types
+
+import flowsint.enricher.abc as _fi_enricher_abc
+
+
+class Enricher(_fi_enricher_abc.EnricherABC):
+    # Define types as class attributes (base types, not lists)
+    InputType = _fi_types.Domain
+    OutputType = _fi_types.Ip
+
+    @classmethod
+    def name(cls):
+        return "my_enricher"
+
+    @classmethod
+    def category(cls):
+        return "Domain"
+
+    @classmethod
+    def key(cls):
+        return "domain"
+
+    # preprocess receives a list and returns a list of validated InputType instances
+    def preprocess(self, data: _t.List) -> _t.List[_fi_types.Domain]:
+        # Generic implementation handles validation automatically
+        return super().preprocess(data)
+
+    # scan receives a list of InputType and returns a list of OutputType
+    async def scan(self, data: _t.List[_fi_types.Domain]) -> _t.List[_fi_types.Ip]:
+        results: _t.List[_fi_types.Ip] = []
+        # ... implementation
+        return results
+
+# Make types available at module level for easy access
+InputType = Enricher.InputType
+OutputType = Enricher.OutputType

python/packages/flowsint-types/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "flowsint-types-next"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+  "flowsint-types",
+]
+
+[build-system]
+requires = ["uv_build >= 0.10.7, <0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "flowsint.types"
+
+[tool.uv.sources]
+flowsint-types = { path = "../../../flowsint-types", editable = true }

python/packages/flowsint-types/src/flowsint/types/__init__.py

@@ -0,0 +1 @@
+from flowsint_types import *

python/packages/flowsint/src/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

python/packages/flowsint/src/__main__.py

@@ -0,0 +1 @@
+print('root flowsint __main__.py')