ref(resolver)

Author: xiaoland

app/business/info_base/block.py

@@ -72,7 +72,7 @@ def get_resolver(cls, block_id: BlockID) -> Opt["Resolver"]:
     block = cls.get(block_id)
     if block is None:
       return None
-    return ResolverManager.new_resolver(block)
+    return ResolverManager.get(block)
 
   @classmethod
   def create(
@@ -111,6 +111,7 @@ def create(
   async def refresh_embeddings(cls):
     """Rebuild all blocks' embeddings - delegates to sink embedding service"""
     from app.business.sink.embedding import EmbeddingManager
+
     await EmbeddingManager.refresh_all_block_embeddings()
 
   @classmethod
@@ -120,8 +121,8 @@ async def fetchsert(cls, block: BlockModel, db_session: sqlmodel.Session) -> Blo
     Will NOT commit the session.
     """
     from app.business.sink.embedding import EmbeddingManager
-    
-    resolver = ResolverManager.new_resolver(block)
+
+    resolver = ResolverManager.get(block)
     existing = resolver.get_existing(db_session)
     if existing is not None:
       logger.debug(
@@ -149,7 +150,7 @@ async def organize(cls, block: BlockModel):
     FIXME
     """
     with SessionLocal() as db_session:
-      resolver = ResolverManager.new_resolver(block)
+      resolver = ResolverManager.get(block)
       generator = (await resolver.breakdown())()
       try:
         i = generator.send(None)
@@ -179,6 +180,7 @@ def query_by_embedding(
     :param resolver: Filter by resolver type, None means no filter
     """
     from app.business.sink.embedding import EmbeddingManager
+
     return EmbeddingManager.query_blocks_by_embedding(
       block_id=block_id,
       embedding=embedding,

app/business/info_base/resolver/main.py

@@ -4,32 +4,33 @@
 
 import sqlmodel
 
+from app.business.info_base.relation import RelationManager
 from app.business.info_base.storage.main import StorageManager
 
 from app.schemas.info_base.main import SubGraphForm
-from app.schemas.info_base.block import ResolverType, BlockModel
+from app.schemas.info_base.block import BlockID, ResolverType, BlockModel
 from app.schemas.info_base.relation import RelationModel
 from app.schemas.info_base.storage import StorageID
 
 
 class ResolverManager:
-  RESOLVERS: dict[ResolverType, type["Resolver"]] = {}
+  RESOLVER_CLS: dict[ResolverType, type["Resolver"]] = {}
   """Global resolver registry.
 
-    Map ResolverType to Resolver class
-    """
+  Map ResolverType to Resolver class
+  """
 
   @classmethod
   def register_resolver(cls, resolver_cls: type["Resolver"]):
-    cls.RESOLVERS[resolver_cls.__rsotype__] = resolver_cls
+    cls.RESOLVER_CLS[resolver_cls.__rsotype__] = resolver_cls
 
   @classmethod
-  def new_resolver(cls, block: BlockModel) -> "Resolver":
+  def get(cls, block: BlockModel) -> "Resolver":
     """Create resolver instance from block."""
     try:
-      resolver_cls = cls.RESOLVERS[block.resolver]
+      resolver_cls = cls.RESOLVER_CLS[block.resolver]
     except KeyError:
-      raise NotImplementedError(f"Resolver {block.resolver} not implemented.")
+      raise NotImplementedError(f"Resolver {block.resolver} not implemented/registered.")
     return resolver_cls(block)
 
 
@@ -38,7 +39,12 @@ def new_resolver(cls, block: BlockModel) -> "Resolver":
 
 
 class Resolver(abc.ABC, typing.Generic[SolvedContentTV, RawContentTV]):
-  """Resolver resolves a star graph (a block and its direct relations)"""
+  """Resolver resolves a star graph (a block and its direct relations)
+
+  Generic parameters:
+  - SolvedContentTV: The type of the solved content
+  - RawContentTV: The type of the raw content
+  """
 
   __rsotype__: ResolverType
   """Resolver type"""
@@ -55,18 +61,67 @@ def __init__(self, block: BlockModel, relations: Opt[tuple[RelationModel, ...]]
     :param relations: Relations of the block.
     """
     self._block = block
-    self._relations = relations or tuple()
-    self._raw_content: RawContentTV
-    self._solved_content: SolvedContentTV
-    self.__post_init__()
+    self.__relations = relations or None
+    self.__raw_content: RawContentTV | None = None
+    """The (real) content of the block, commonly fetched from storage. 
+    If storage is None, uses block.content
+    """
+    if self._block.storage is None:
+      self.__raw_content = typing.cast(RawContentTV, self._block.content)
+    self.__solved_content: SolvedContentTV | None = None
+    """Solved content is the content the resolver really works with,
+    commonly from raw content.
+    """
+    self.__post_init__(self.__raw_content)
 
-  def __post_init__(self):
+  def __post_init__(self, raw_content: Opt[RawContentTV] = None) -> None:
     """Subclass post-initialization hook.
 
-    It's suggest to resolve the block content to _solved_content here.
+    It's suggest to set __solved_content here if possible:
+    ```python
+    async def __post_init__(self, raw_content):
+      if raw_content is not None:
+        ... # anyhow from raw_content
+        self.set_solved_content(solved_content)
+    ```
     """
     ...
 
+  @property
+  def block_id(self) -> BlockID:
+    """Get the block ID."""
+    return typing.cast(BlockID, self._block.id)
+
+  async def get_raw_content(self) -> RawContentTV:
+    """Get the raw content of the block."""
+    if self.__raw_content is None:
+      if self._block.storage is None:
+        self.__raw_content = typing.cast(RawContentTV, self._block.content)
+      else:
+        storage = StorageManager.get_storage(self._block.storage)
+        self.__raw_content = typing.cast(
+          RawContentTV, await storage.get_raw_content(self._block.content)
+        )
+
+    return self.__raw_content
+
+  async def get_solved_content(self) -> SolvedContentTV:
+    """Get the solved content of the block."""
+    if self.__solved_content is None:
+      raise NotImplementedError(
+        f"{self.__class__.__name__}.get_solved_content() must be implemented by subclasses."
+      )
+    return self.__solved_content
+
+  def set_solved_content(self, content: SolvedContentTV) -> None:
+    self.__solved_content = content
+
+  async def get_relations(self) -> tuple[RelationModel, ...]:
+    """Get relations of the block."""
+    if self.__relations is None:
+      self.__relations = RelationManager.get(block_id=self.block_id)
+    return self.__relations
+
   @classmethod
   # @abc.abstractmethod TODO
   def create_block(cls, content, storage: Opt[StorageID] = None) -> BlockModel: ...
@@ -75,14 +130,15 @@ def create_block(cls, content, storage: Opt[StorageID] = None) -> BlockModel: ..
   # @abc.abstractmethod TODO
   def create_graph(cls, *args, **kwargs) -> SubGraphForm: ...
 
+  @abc.abstractmethod
   async def get_text(self) -> str:
     """Get block content in text format."""
-    storage = StorageManager.new_storage(self._block)
-    return await storage.get_content(self._block)
+    ...
 
-  def get_str_for_embedding(self) -> str:
+  @abc.abstractmethod
+  async def get_str_for_embedding(self) -> str:
     """Get string representation for embedding generation."""
-    return self._block.content
+    ...
 
   def get_existing(self, db_session: sqlmodel.Session) -> Opt[BlockModel]:
     """Check if a block with the same content already exists in the database.

app/business/info_base/storage/http.py

@@ -47,7 +47,7 @@ async def _fetch_url(self, url: str) -> aiohttp.ClientResponse:
         response.raise_for_status()
         return response
 
-  async def get_content(self, block: BlockModel) -> str | bytes:
+  async def get_raw_content(self, block: BlockModel) -> str | bytes:
     """Fetch content from the URL in block.content.
 
     :param block: Block containing the URL in its content field
@@ -59,7 +59,7 @@ async def get_content(self, block: BlockModel) -> str | bytes:
 class HTTPImageStorage(HTTPStorage):
   """HTTP storage for image content (returns base64-encoded data)."""
 
-  async def get_content(self, block: BlockModel) -> str:
+  async def get_raw_content(self, block: BlockModel) -> str:
     """Fetch and encode image content as base64."""
     url = block.content
     config = self.get_config()
@@ -76,7 +76,7 @@ async def get_content(self, block: BlockModel) -> str:
 class HTTPVideoStorage(HTTPStorage):
   """HTTP storage for video content (returns raw bytes)."""
 
-  async def get_content(self, block: BlockModel) -> bytes:
+  async def get_raw_content(self, block: BlockModel) -> bytes:
     """Fetch video content as bytes."""
     url = block.content
     config = self.get_config()
@@ -93,7 +93,7 @@ async def get_content(self, block: BlockModel) -> bytes:
 class HTTPTextStorage(HTTPStorage):
   """HTTP storage for plain text content."""
 
-  async def get_content(self, block: BlockModel) -> str:
+  async def get_raw_content(self, block: BlockModel) -> str:
     """Fetch text content from URL."""
     url = block.content
     config = self.get_config()
@@ -109,7 +109,7 @@ async def get_content(self, block: BlockModel) -> str:
 class HTTPJsonStorage(HTTPStorage):
   """HTTP storage for JSON content."""
 
-  async def get_content(self, block: BlockModel) -> str:
+  async def get_raw_content(self, block: BlockModel) -> str:
     """Fetch JSON content from URL."""
     url = block.content
     config = self.get_config()
@@ -125,7 +125,7 @@ async def get_content(self, block: BlockModel) -> str:
 class HTTPBinaryStorage(HTTPStorage):
   """HTTP storage for binary content (returns raw bytes)."""
 
-  async def get_content(self, block: BlockModel) -> bytes:
+  async def get_raw_content(self, block: BlockModel) -> bytes:
     """Fetch binary content as bytes."""
     url = block.content
     config = self.get_config()
@@ -142,7 +142,7 @@ async def get_content(self, block: BlockModel) -> bytes:
 class HTTPHtmlStorage(HTTPStorage):
   """HTTP storage for HTML content (returns raw bytes)."""
 
-  async def get_content(self, block: BlockModel) -> bytes:
+  async def get_raw_content(self, block: BlockModel) -> bytes:
     """Fetch HTML content as bytes."""
     url = block.content
     config = self.get_config()

app/business/info_base/storage/main.py

@@ -59,7 +59,7 @@ def register_storage(cls, storage_cls: type["Storage"]):
     cls._STORAGE_CLASSES[storage_type] = storage_cls
 
   @classmethod
-  def _get_storage_ins(cls, storage_id: StorageID) -> "Storage":
+  def get_storage(cls, storage_id: StorageID) -> "Storage":
     """Create a storage instance."""
     with SessionLocal() as db:
       storage_record = db.exec(
@@ -74,15 +74,6 @@ def _get_storage_ins(cls, storage_id: StorageID) -> "Storage":
 
     return storage_class(storage_record)
 
-  @classmethod
-  def new_storage(cls, block: BlockModel) -> "Storage":
-    """Create storage instance from block."""
-    if block.storage is None:
-      return Storage(None)
-
-    storage_id = typing.cast(StorageID, block.storage)
-    return cls._get_storage_ins(storage_id)
-
   @classmethod
   def create(
     cls,
@@ -199,32 +190,18 @@ def __init_subclass__(cls, config_cls: type[ConfigTV], **kwargs) -> None:
     StorageManager.register_storage(cls)
     return super().__init_subclass__(**kwargs)
 
-  def __init__(self, storage_record: Opt[StorageModel]):
-    if storage_record is not None:
-      self._type = storage_record.type
-      self._config = storage_record.config
-    else:
-      self._type = None  # Inline content storage
-      self._config = {}
+  def __init__(self, storage_record: StorageModel):
+    self._type = storage_record.type
+    self._config = storage_record.config
 
     self.__post_init__()
 
   def __post_init__(self):
     """Post-initialization hook for subclasses."""
     pass
 
-  async def get_content(self, block_content: str) -> ContentTV:
-    """Get the actual content of the block."""
-    if self._type is None:
-      # Inline content storage
-      return typing.cast(ContentTV, block_content)
-    return await self._get_content(block_content)
-
-  async def _get_content(self, block_content: str) -> ContentTV:
-    """Get the actual content from the concrete storage implementation.
-
-    Subclasses must override this method.
-    """
+  async def get_raw_content(self, block_content: str) -> ContentTV:
+    """Get the raw content of the block."""
     raise NotImplementedError(
-      f"{self.__class__.__name__}._get_content() must be implemented by subclasses."
+      f"{self.__class__.__name__}.get_raw_content() must be implemented by subclasses."
     )

app/business/sink/embedding.py

@@ -47,10 +47,12 @@ async def upsert_block_embedding(
         if block is None:
           raise ValueError(f"Block with id {block_id} not found")
 
-    resolver = ResolverManager.new_resolver(block)
+    resolver = ResolverManager.get(block)
     embedding = BlockEmbeddingModel(
       id=block.id,  # type: ignore[arg-type]
-      embedding=Embedding("", "text-embedding-v3").embed(resolver.get_str_for_embedding()),
+      embedding=Embedding("", "text-embedding-v3").embed(
+        await resolver.get_str_for_embedding()
+      ),
     )
     if db_session:
       db_session.merge(embedding)

app/schemas/info_base/block.py

@@ -60,5 +60,5 @@ class BlockModel(sqlmodel.SQLModel, table=True):
   async def get_context_as_text(self) -> str:
     from app.business.info_base.resolver import ResolverManager
 
-    resolver = ResolverManager.new_resolver(self)
+    resolver = ResolverManager.get(self)
     return await resolver.get_text()

docs/todo.md

@@ -44,8 +44,7 @@
 - [x] Resolver relies on Storage to get the actual content (don't do it yourself, never considering what storage is)
 - [x] 改进加载模式
   - 在未找到时，按照类型（和 Python 导入路径语法一致）尝试从插件中导入 （否则插件就需要在初始化时导入）
-- [ ] Resolver 和 Storage 应该解耦；比如 ImageResolver 要的就是 content 为图片二进制的 block，storage就负责搞定这件事，resolver不应该在乎
-  - 
+- [ ] 规范化 Resolver，其负责解析 Block 的 StarGraph
 
 ## Sink
 

extensions/github/resolver.py

@@ -7,17 +7,17 @@
 from app.business.info_base.resolver import Resolver
 from app.schemas.info_base.block import BlockModel
 from app.schemas.info_base.relation import RelationModel
-from app.schemas.info_base.main import ArcForm, SubGraphForm
+from app.schemas.info_base.main import InArcForm, SubGraphForm
 from utils.sql import find_by_json_contains
 from .schema import GithubRepo, GithubUser
 
 
 class GithubRepoResolver(Resolver, rso_type="github_repo"):
   """Resolver for GitHub repository blocks."""
 
-  def __post_init__(self):
-    """Parse GitHub repo content after initialization."""
-    self.content = GithubRepo.model_validate_json(self._block.content)
+  def __post_init__(self, raw_content):
+    if raw_content is not None:
+      self.set_solved_content(GithubRepo.model_validate_json(raw_content))
 
   @classmethod
   def create_graph(
@@ -38,7 +38,7 @@ def create_graph(
     in_relations = ()
     if owner:
       in_relations = (
-        ArcForm(
+        InArcForm(
           relation=RelationModel(content="owns"),
           from_subgraph=GithubUserResolver.create_graph(owner),
         ),
@@ -73,7 +73,7 @@ async def get_text(self) -> str:
       text += f": {self.content.description}"
     return text
 
-  def get_str_for_embedding(self) -> str:
+  async def get_str_for_embedding(self) -> str:
     """Get text for embedding generation.
 
     Combines name, description, topics and language for better semantic search.