diff --git a/memory/memory_extractor.py b/memory/memory_extractor.py index d488da4..15eef82 100644 --- a/memory/memory_extractor.py +++ b/memory/memory_extractor.py @@ -22,7 +22,7 @@ from core.logging_manager import get_logger from .toml_tree_store import TomlTreeStore, Memory -from .memory_index import MemoryIndex +from .memory_index import MemoryIndex, normalize_tags from .llm_adapter import chat_text logger = get_logger("memory_extractor", "green") @@ -532,10 +532,13 @@ async def deduplicate_and_store( matched.importance = max(importance, matched.importance) matched.meta["last_accessed"] = time.time() - # 合并 tags - existing_tags = set(matched.tags) - existing_tags.update(tags) - matched.tags = list(existing_tags) + # 合并 tags —— 两边都先规整再做集合运算。`set()` 吃到 TOML 里 + # 手工写进去的 dict/list 会抛 unhashable TypeError，而这个循环 + # 外面只有 hippocampus_process 的一个大 try，异常会把本批剩余 + # facts 连同升维、画像更新一起吞掉，不是只丢一个 tag。 + existing_tags = set(normalize_tags(matched.tags)) + existing_tags.update(normalize_tags(tags)) + matched.tags = sorted(existing_tags) if await self.tree_store.update_memory(matched): logger.info(f"Memory merged: id={matched.id}") diff --git a/memory/memory_index.py b/memory/memory_index.py index 2f8f0c5..8686c6c 100644 --- a/memory/memory_index.py +++ b/memory/memory_index.py @@ -30,6 +30,28 @@ logger = get_logger("kiraos_memory_index", "green") + +def normalize_tags(tags) -> list[str]: + """把任意来源的 tags 规整为 list[str]（去掉 null / 非字符串 / 空白项）。 + + 三个写入路径共用这一份规则，避免 TOML 与 SQLite 各写一套导致同一条 + memory 在两侧存出不同的 tag 集合： + + - `MemoryIndex.upsert()` —— LLM 提取出的数组可能混进 `null`/数字， + `json.dumps` 能过但 `" ".join` 会抛 TypeError 打掉整条写入。 + - `MemoryIndex.rebuild_from_filesystem()` —— 直接吃手工编辑过的 TOML + 原值，不经过 `Memory.from_toml_dict()` 的类型回退。 + - `TomlTreeStore._sync_write_toml()` —— 只剥 None 的话 `""` 会留在 + TOML 里，而索引侧会丢掉它，重建索引后 tag 集合再次漂移。 + + 非 list 容器（字符串会被逐字符迭代、dict 会被迭代成 key）整体视为空， + 这类值本身就是坏数据，逐字符拆开只会写进更多垃圾 tag。 + """ + if not isinstance(tags, list): + return [] + return [t for t in tags if isinstance(t, str) and t.strip()] + + # jieba 是中文分词的最佳选择，但不应该作为插件加载的硬依赖。 # 没装的话，降级为按字符切分 + ASCII whitespace —— FTS5 仍可工作， # 只是中文查准率会差一些。运行时会打一次明显的 warning 提醒。 @@ -327,13 +349,14 @@ def upsert( if not last_accessed: last_accessed = now - tags_json = json.dumps(tags or [], ensure_ascii=False) + tags = normalize_tags(tags) + tags_json = json.dumps(tags, ensure_ascii=False) source_json = json.dumps(source or {}, ensure_ascii=False) chash = self.content_hash(raw_text) # jieba 分词后存入 FTS（确保中文可检索） segmented_text = self._segment_for_fts(raw_text) - tags_flat = " ".join(tags or []) + tags_flat = " ".join(tags) with self._transaction() as cur: cur.execute(""" @@ -924,7 +947,7 @@ def bulk_upsert(self, records: List[Dict[str, Any]]): entity_type = rec.get("entity_type", "") folder = rec.get("folder", "facts") base_dir = rec.get("base_dir", "") - tags = rec.get("tags", []) + tags = normalize_tags(rec.get("tags", [])) tags_json = json.dumps(tags, ensure_ascii=False) source_json = json.dumps(rec.get("source", {}), ensure_ascii=False) raw_text = rec.get("raw_text", "") diff --git a/memory/toml_tree_store.py b/memory/toml_tree_store.py index 223e5f4..4ba6179 100644 --- a/memory/toml_tree_store.py +++ b/memory/toml_tree_store.py @@ -45,7 +45,7 @@ get_entity_folder, ensure_entity_dirs, ) -from .memory_index import MemoryIndex +from .memory_index import MemoryIndex, normalize_tags logger = get_logger("kiraos_toml_tree_store", "green") @@ -987,9 +987,21 @@ def _atomic_write_bytes(fpath: str, payload: bytes) -> None: @classmethod def _sync_write_toml(cls, memory: Memory): - """写入 TOML 文件（人类可读内容，无运行时 meta）；原子替换。""" + """写入 TOML 文件（人类可读内容，无运行时 meta）；原子替换。 + + tags 走 `normalize_tags` —— 和 `MemoryIndex.upsert()` 共用同一份规则。 + 只靠 `_clean_for_toml` 剥 None 的话，`""` / 纯空白 tag 会留在 TOML 里 + 而索引侧会丢掉，同一次 update_memory 在两边存出不同的 tag 集合，读取 + 或重建索引后结果还会再变一次。 + + `_clean_for_toml` 仍然保留：它负责 source 等其它字段里的 None， + `tomli_w.dumps` 遇到 None 会抛 "Object of type 'NoneType' is not TOML + serializable"，让整条 update_memory 返回 False（合并静默失败）。 + """ fpath = memory.file_path data = memory.to_toml_dict() + data["tags"] = normalize_tags(data.get("tags")) + data = _clean_for_toml(data) cls._atomic_write_bytes(fpath, tomli_w.dumps(data).encode("utf-8")) @staticmethod diff --git a/tests/test_storage_layer.py b/tests/test_storage_layer.py index 4731ce3..6c16444 100644 --- a/tests/test_storage_layer.py +++ b/tests/test_storage_layer.py @@ -169,6 +169,78 @@ async def run(): _run(run()) +def test_update_memory_survives_null_tags_from_llm(store): + """A null in the tags array must not break the merge write path. + + The hippocampus merge path unions LLM-extracted tags into the matched + memory. An LLM that emits `null` inside that array used to blow up both + writers -- tomli_w on the TOML side, `" ".join` on the index side -- so + update_memory returned False and the merge was silently lost. + """ + async def run(): + mem = await store.add_memory( + content_text="周武住在杭州", + memory_type="fact", + importance=6, + tags=["location"], + entity_id="qq:769690776", + entity_type="user", + folder="facts", + ) + + mem.text = "周武住在杭州，最近搬到了西湖区" + mem.tags = ["location", None, "", " ", 42, "city"] + + assert await store.update_memory(mem) is True + + reread = await store.get_memory( + mem.id, "qq:769690776", "user", "facts" + ) + assert reread is not None + assert reread.text == "周武住在杭州，最近搬到了西湖区" + # 规整只过滤、不排序，原有顺序保留。 + assert reread.tags == ["location", "city"] + + _run(run()) + + +def test_toml_and_index_agree_on_normalized_tags(store): + """The TOML file and the SQLite index must store the same tag set. + + `_clean_for_toml` only drops None, so an empty-string tag used to survive + into the TOML while `MemoryIndex.upsert()` filtered it out. That left the + two writers disagreeing about one memory's tags, and rebuilding the index + from the TOML shifted the set a second time. + """ + async def run(): + mem = await store.add_memory( + content_text="周武喜欢西湖", + memory_type="fact", + importance=6, + tags=["place"], + entity_id="qq:769690776", + entity_type="user", + folder="facts", + ) + + mem.tags = ["place", "", None, "lake"] + assert await store.update_memory(mem) is True + + raw = TomlTreeStore._sync_read_toml(mem.file_path) + assert raw["tags"] == ["place", "lake"] + + indexed = store.index.get_meta( + mem.id, + entity_id="qq:769690776", + entity_type="user", + folder="facts", + ) + assert indexed is not None + assert sorted(indexed["tags"]) == ["lake", "place"] + + _run(run()) + + def test_content_hash_is_namespaced_per_entity(store): """The same sentence about two different people is two memories.""" async def run():