From 84e856b434f1ee7ce2e656f140ad6b8cde09c2af Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Sat, 29 Nov 2025 14:57:00 +0800 Subject: [PATCH 01/13] function call supoort --- src/memos/mem_reader/multi_model_struct.py | 2 +- .../read_multi_model/system_parser.py | 64 +++++++++++++++---- .../read_multi_model/tool_parser.py | 14 +++- src/memos/memories/textual/item.py | 11 +++- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/memos/mem_reader/multi_model_struct.py b/src/memos/mem_reader/multi_model_struct.py index 4520058b9..650587472 100644 --- a/src/memos/mem_reader/multi_model_struct.py +++ b/src/memos/mem_reader/multi_model_struct.py @@ -75,7 +75,7 @@ def _process_multi_model_data( fast_memory_items = self._concat_multi_model_memories(all_memory_items) else: - # Parse as single message + # Parse as single string fast_memory_items = self.multi_model_parser.parse( scene_data_info, info, mode="fast", **kwargs ) diff --git a/src/memos/mem_reader/read_multi_model/system_parser.py b/src/memos/mem_reader/read_multi_model/system_parser.py index 258b752cc..97d3f1efa 100644 --- a/src/memos/mem_reader/read_multi_model/system_parser.py +++ b/src/memos/mem_reader/read_multi_model/system_parser.py @@ -1,14 +1,22 @@ """Parser for system messages.""" +import json +import re +import uuid + from typing import Any from memos.embedders.base import BaseEmbedder from memos.llms.base import BaseLLM from memos.log import get_logger -from memos.memories.textual.item import SourceMessage, TextualMemoryItem +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) from memos.types.openai_chat_completion_types import ChatCompletionSystemMessageParam -from .base import BaseMessageParser, _extract_text_from_content +from .base import BaseMessageParser logger = get_logger(__name__) @@ -29,20 +37,19 @@ def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None): def create_source( self, - message: ChatCompletionSystemMessageParam, + message: str, info: dict[str, Any], ) -> SourceMessage: """Create SourceMessage from system message.""" - if not isinstance(message, dict): - return SourceMessage(type="chat", role="system") + tool_schema_match = re.search(r"(.*?)", message, re.DOTALL) + tool_schema_content = tool_schema_match.group(1) if tool_schema_match else "" - content = _extract_text_from_content(message.get("content", "")) return SourceMessage( type="chat", role="system", - chat_time=message.get("chat_time"), - message_id=message.get("message_id"), - content=content, + chat_time=message.get("chat_time", None), + message_id=message.get("message_id", None), + content=tool_schema_content, ) def rebuild_from_source( @@ -63,7 +70,25 @@ def parse_fast( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - return super().parse_fast(message, info, **kwargs) + content = message["content"] + if isinstance(content, dict): + content = content["text"] + + # Extract tool_schema content and remaining content + content_wo_tool_schema = re.sub( + r"(.*?)", + r"omitted", + content, + flags=re.DOTALL, + ) + + source = self.create_source(content, info) + return [ + TextualMemoryItem( + memory=content_wo_tool_schema, + metadata=TreeNodeTextualMemoryMetadata(sources=[source]), + ) + ] def parse_fine( self, @@ -71,4 +96,21 @@ def parse_fine( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - return [] + content = message["content"] + if isinstance(content, dict): + content = content["text"] + try: + tool_schema = json.loads(content) + assert isinstance(tool_schema, list), "Tool schema must be a list" + except json.JSONDecodeError: + return [] + + return [ + TextualMemoryItem( + id=str(uuid.uuid4()), + memory=json.dumps(tool_schema), + metadata=TreeNodeTextualMemoryMetadata( + memory_type="tool_schema", + ), + ) + ] diff --git a/src/memos/mem_reader/read_multi_model/tool_parser.py b/src/memos/mem_reader/read_multi_model/tool_parser.py index f7437312d..4aa17a93e 100644 --- a/src/memos/mem_reader/read_multi_model/tool_parser.py +++ b/src/memos/mem_reader/read_multi_model/tool_parser.py @@ -5,7 +5,11 @@ from memos.embedders.base import BaseEmbedder from memos.llms.base import BaseLLM from memos.log import get_logger -from memos.memories.textual.item import SourceMessage, TextualMemoryItem +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) from memos.types.openai_chat_completion_types import ChatCompletionToolMessageParam from .base import BaseMessageParser, _extract_text_from_content @@ -64,7 +68,13 @@ def parse_fast( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - return super().parse_fast(message, info, **kwargs) + memory = "" + source = self.create_source(message, info) + return [ + TextualMemoryItem( + memory=memory, metadata=TreeNodeTextualMemoryMetadata(sources=[source]) + ) + ] def parse_fine( self, diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 12be08057..76b252f54 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -99,9 +99,14 @@ def __str__(self) -> str: class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata): """Extended metadata for structured memory, layered retrieval, and lifecycle tracking.""" - memory_type: Literal["WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"] = Field( - default="WorkingMemory", description="Memory lifecycle type." - ) + memory_type: Literal[ + "WorkingMemory", + "LongTermMemory", + "UserMemory", + "OuterMemory", + "ToolSchema", + "ToolTrajectory", + ] = Field(default="WorkingMemory", description="Memory lifecycle type.") sources: list[SourceMessage] | None = Field( default=None, description="Multiple origins of the memory (e.g., URLs, notes)." ) From ad86322912b51c69ff4c9d76b16e21158903a5f0 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Sun, 30 Nov 2025 20:17:53 +0800 Subject: [PATCH 02/13] add tool parser --- .../read_multi_model/tool_parser.py | 113 ++++++++++++++++-- 1 file changed, 101 insertions(+), 12 deletions(-) diff --git a/src/memos/mem_reader/read_multi_model/tool_parser.py b/src/memos/mem_reader/read_multi_model/tool_parser.py index 4aa17a93e..8d42b69a7 100644 --- a/src/memos/mem_reader/read_multi_model/tool_parser.py +++ b/src/memos/mem_reader/read_multi_model/tool_parser.py @@ -1,5 +1,7 @@ """Parser for tool messages.""" +import json + from typing import Any from memos.embedders.base import BaseEmbedder @@ -12,7 +14,7 @@ ) from memos.types.openai_chat_completion_types import ChatCompletionToolMessageParam -from .base import BaseMessageParser, _extract_text_from_content +from .base import BaseMessageParser logger = get_logger(__name__) @@ -35,18 +37,71 @@ def create_source( self, message: ChatCompletionToolMessageParam, info: dict[str, Any], - ) -> SourceMessage: + ) -> SourceMessage | list[SourceMessage]: """Create SourceMessage from tool message.""" + if not isinstance(message, dict): - return SourceMessage(type="chat", role="tool") - - content = _extract_text_from_content(message.get("content", "")) - return SourceMessage( - type="chat", - role="tool", - chat_time=message.get("chat_time"), - message_id=message.get("message_id"), - content=content, + return [] + + role = message.get("role", "tool") + raw_content = message.get("content", "") + tool_call_id = message.get("tool_call_id", "") + chat_time = message.get("chat_time") + message_id = message.get("message_id") + + sources = [] + + if isinstance(raw_content, list): + # Multimodal: create one SourceMessage per part + for part in raw_content: + if isinstance(part, dict): + part_type = part.get("type", "") + if part_type == "file": + file_info = part.get("file", {}) + sources.append( + SourceMessage( + type="file", + role=role, + chat_time=chat_time, + message_id=message_id, + doc_path=file_info.get("filename") or file_info.get("file_id", ""), + content=file_info.get("file_data", ""), + tool_call_id=tool_call_id, + original_part=part, + ) + ) + else: + # image_url, input_audio, etc. + sources.append( + SourceMessage( + type=part_type, + role=role, + chat_time=chat_time, + message_id=message_id, + content=f"[{part_type}]", + tool_call_id=tool_call_id, + original_part=part, + ) + ) + else: + # Simple string content message: single SourceMessage + content = raw_content + if content: + sources.append( + SourceMessage( + type="chat", + role=role, + chat_time=chat_time, + message_id=message_id, + content=content, + tool_call_id=tool_call_id, + ) + ) + + return ( + sources + if len(sources) > 1 + else (sources[0] if sources else SourceMessage(type="chat", role=role)) ) def rebuild_from_source( @@ -54,6 +109,40 @@ def rebuild_from_source( source: SourceMessage, ) -> ChatCompletionToolMessageParam: """Rebuild tool message from SourceMessage.""" + + # Priority 1: Use original_part if available + if hasattr(source, "original_part") and source.original_part: + original = source.original_part + # If it's a content part, wrap it in a message + if isinstance(original, dict) and "type" in original: + return { + "role": source.role or "user", + "content": [original], + "chat_time": source.chat_time, + "message_id": source.message_id, + } + # If it's already a full message, return it + if isinstance(original, dict) and "role" in original: + return original + + # Priority 2: Rebuild from source fields + if source.type == "file": + return { + "role": source.role or "user", + "content": [ + { + "type": "file", + "file": { + "filename": source.doc_path or "", + "file_data": source.content or "", + }, + } + ], + "chat_time": source.chat_time, + "message_id": source.message_id, + } + + # Simple text message return { "role": "tool", "content": source.content or "", @@ -68,7 +157,7 @@ def parse_fast( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - memory = "" + memory = json.dumps(message) source = self.create_source(message, info) return [ TextualMemoryItem( From e3aaf689f3b7269c232ee6fb02a801e16f0ad2a4 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Mon, 1 Dec 2025 13:59:31 +0800 Subject: [PATCH 03/13] rename multi model to modal --- .../mem_reader/{multi_model_struct.py => multi_modal_struct.py} | 0 .../mem_reader/{read_multi_model => read_multi_modal}/__init__.py | 0 .../{read_multi_model => read_multi_modal}/assistant_parser.py | 0 .../mem_reader/{read_multi_model => read_multi_modal}/base.py | 0 .../{read_multi_model => read_multi_modal}/file_content_parser.py | 0 .../{read_multi_model => read_multi_modal}/multi_model_parser.py | 0 .../{read_multi_model => read_multi_modal}/string_parser.py | 0 .../{read_multi_model => read_multi_modal}/system_parser.py | 0 .../{read_multi_model => read_multi_modal}/text_content_parser.py | 0 .../{read_multi_model => read_multi_modal}/tool_parser.py | 0 .../{read_multi_model => read_multi_modal}/user_parser.py | 0 .../mem_reader/{read_multi_model => read_multi_modal}/utils.py | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename src/memos/mem_reader/{multi_model_struct.py => multi_modal_struct.py} (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/__init__.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/assistant_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/base.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/file_content_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/multi_model_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/string_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/system_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/text_content_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/tool_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/user_parser.py (100%) rename src/memos/mem_reader/{read_multi_model => read_multi_modal}/utils.py (100%) diff --git a/src/memos/mem_reader/multi_model_struct.py b/src/memos/mem_reader/multi_modal_struct.py similarity index 100% rename from src/memos/mem_reader/multi_model_struct.py rename to src/memos/mem_reader/multi_modal_struct.py diff --git a/src/memos/mem_reader/read_multi_model/__init__.py b/src/memos/mem_reader/read_multi_modal/__init__.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/__init__.py rename to src/memos/mem_reader/read_multi_modal/__init__.py diff --git a/src/memos/mem_reader/read_multi_model/assistant_parser.py b/src/memos/mem_reader/read_multi_modal/assistant_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/assistant_parser.py rename to src/memos/mem_reader/read_multi_modal/assistant_parser.py diff --git a/src/memos/mem_reader/read_multi_model/base.py b/src/memos/mem_reader/read_multi_modal/base.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/base.py rename to src/memos/mem_reader/read_multi_modal/base.py diff --git a/src/memos/mem_reader/read_multi_model/file_content_parser.py b/src/memos/mem_reader/read_multi_modal/file_content_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/file_content_parser.py rename to src/memos/mem_reader/read_multi_modal/file_content_parser.py diff --git a/src/memos/mem_reader/read_multi_model/multi_model_parser.py b/src/memos/mem_reader/read_multi_modal/multi_model_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/multi_model_parser.py rename to src/memos/mem_reader/read_multi_modal/multi_model_parser.py diff --git a/src/memos/mem_reader/read_multi_model/string_parser.py b/src/memos/mem_reader/read_multi_modal/string_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/string_parser.py rename to src/memos/mem_reader/read_multi_modal/string_parser.py diff --git a/src/memos/mem_reader/read_multi_model/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/system_parser.py rename to src/memos/mem_reader/read_multi_modal/system_parser.py diff --git a/src/memos/mem_reader/read_multi_model/text_content_parser.py b/src/memos/mem_reader/read_multi_modal/text_content_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/text_content_parser.py rename to src/memos/mem_reader/read_multi_modal/text_content_parser.py diff --git a/src/memos/mem_reader/read_multi_model/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/tool_parser.py rename to src/memos/mem_reader/read_multi_modal/tool_parser.py diff --git a/src/memos/mem_reader/read_multi_model/user_parser.py b/src/memos/mem_reader/read_multi_modal/user_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/user_parser.py rename to src/memos/mem_reader/read_multi_modal/user_parser.py diff --git a/src/memos/mem_reader/read_multi_model/utils.py b/src/memos/mem_reader/read_multi_modal/utils.py similarity index 100% rename from src/memos/mem_reader/read_multi_model/utils.py rename to src/memos/mem_reader/read_multi_modal/utils.py From f62a010ecd68d23086dc269842c716725891ca11 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Mon, 1 Dec 2025 14:17:14 +0800 Subject: [PATCH 04/13] rename multi modal --- .../{multi_model_parser.py => multi_modal_parser.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/memos/mem_reader/read_multi_modal/{multi_model_parser.py => multi_modal_parser.py} (100%) diff --git a/src/memos/mem_reader/read_multi_modal/multi_model_parser.py b/src/memos/mem_reader/read_multi_modal/multi_modal_parser.py similarity index 100% rename from src/memos/mem_reader/read_multi_modal/multi_model_parser.py rename to src/memos/mem_reader/read_multi_modal/multi_modal_parser.py From 87091d6409d77b29b171f5c9b997c6b7d9a2b4af Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Mon, 1 Dec 2025 21:12:48 +0800 Subject: [PATCH 05/13] tool mem support --- src/memos/mem_reader/multi_modal_struct.py | 70 +++++++- .../read_multi_modal/system_parser.py | 24 ++- .../read_multi_modal/tool_parser.py | 151 +++++++++++++++--- src/memos/mem_reader/simple_struct.py | 2 + src/memos/memories/textual/item.py | 8 +- src/memos/templates/tool_mem_prompts.py | 84 ++++++++++ 6 files changed, 307 insertions(+), 32 deletions(-) create mode 100644 src/memos/templates/tool_mem_prompts.py diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 5a78208b9..7c7ec64de 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -1,4 +1,5 @@ import concurrent.futures +import json import traceback from typing import Any @@ -7,8 +8,9 @@ from memos.configs.mem_reader import MultiModalStructMemReaderConfig from memos.context.context import ContextThreadPoolExecutor from memos.mem_reader.read_multi_modal import MultiModalParser -from memos.mem_reader.simple_struct import SimpleStructMemReader +from memos.mem_reader.simple_struct import SimpleStructMemReader, detect_lang from memos.memories.textual.item import TextualMemoryItem +from memos.templates.tool_mem_prompts import TOOL_TRAJECTORY_PROMPT_EN, TOOL_TRAJECTORY_PROMPT_ZH from memos.types import MessagesType from memos.utils import timed @@ -222,6 +224,61 @@ def _process_string_fine( return fine_memory_items + def _get_llm_tool_trajectory_response(self, mem_str: str) -> dict: + """ + Generete tool trajectory experience item by llm. + """ + try: + lang = detect_lang(mem_str) + template = TOOL_TRAJECTORY_PROMPT_ZH if lang == "zh" else TOOL_TRAJECTORY_PROMPT_EN + prompt = template.replace("{messages}", mem_str) + rsp = self.llm.generate([{"role": "user", "content": prompt}]) + rsp = rsp.replace("```json", "").replace("```", "") + return json.loads(rsp) + except Exception as e: + logger.error(f"[MultiModalFine] Error calling LLM for tool trajectory: {e}") + return [] + + def _process_tool_trajectory_fine( + self, + fast_memory_items: list[TextualMemoryItem], + info: dict[str, Any], + ) -> list[TextualMemoryItem]: + """ + Process tool trajectory memory items through LLM to generate fine mode memories. + """ + if not fast_memory_items: + return [] + + fine_memory_items = [] + + for fast_item in fast_memory_items: + # Extract memory text (string content) + mem_str = fast_item.memory or "" + if not mem_str.strip() or "tool:" not in mem_str: + continue + try: + resp = self._get_llm_tool_trajectory_response(mem_str) + except Exception as e: + logger.error(f"[MultiModalFine] Error calling LLM for tool trajectory: {e}") + continue + for m in resp: + try: + # Normalize memory_type (same as simple_struct) + memory_type = "ToolTrajectoryMemory" + + node = self._make_memory_item( + value=m.get("trajectory", ""), + info=info, + memory_type=memory_type, + tool_used_status=m.get("tool_used_status", []), + ) + fine_memory_items.append(node) + except Exception as e: + logger.error(f"[MultiModalFine] parse error for tool trajectory: {e}") + + return fine_memory_items + @timed def _process_multi_modal_data( self, scene_data_info: MessagesType, info, mode: str = "fine", **kwargs @@ -266,6 +323,11 @@ def _process_multi_modal_data( ) fine_memory_items.extend(fine_memory_items_string_parser) + fine_memory_items_tool_trajectory_parser = self._process_tool_trajectory_fine( + fast_memory_items, info + ) + fine_memory_items.extend(fine_memory_items_tool_trajectory_parser) + # Part B: get fine multimodal items for fast_item in fast_memory_items: sources = fast_item.metadata.sources @@ -304,6 +366,12 @@ def _process_transfer_multi_modal_data( # Part A: call llm fine_memory_items_string_parser = self._process_string_fine([raw_node], info, custom_tags) fine_memory_items.extend(fine_memory_items_string_parser) + + fine_memory_items_tool_trajectory_parser = self._process_tool_trajectory_fine( + [raw_node], info + ) + fine_memory_items.extend(fine_memory_items_tool_trajectory_parser) + # Part B: get fine multimodal items for source in sources: items = self.multi_modal_parser.process_transfer( diff --git a/src/memos/mem_reader/read_multi_modal/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py index 9a7b1b32f..9056a050c 100644 --- a/src/memos/mem_reader/read_multi_modal/system_parser.py +++ b/src/memos/mem_reader/read_multi_modal/system_parser.py @@ -41,6 +41,12 @@ def create_source( info: dict[str, Any], ) -> SourceMessage: """Create SourceMessage from system message.""" + content_wo_tool_schema = re.sub( + r"(.*?)", + r"omitted", + message, + flags=re.DOTALL, + ) tool_schema_match = re.search(r"(.*?)", message, re.DOTALL) tool_schema_content = tool_schema_match.group(1) if tool_schema_match else "" @@ -49,7 +55,8 @@ def create_source( role="system", chat_time=message.get("chat_time", None), message_id=message.get("message_id", None), - content=tool_schema_content, + content=content_wo_tool_schema, + tool_schema=tool_schema_content, ) def rebuild_from_source( @@ -57,9 +64,10 @@ def rebuild_from_source( source: SourceMessage, ) -> ChatCompletionSystemMessageParam: """Rebuild system message from SourceMessage.""" + # only rebuild tool schema content, content will be used in full chat content by llm return { "role": "system", - "content": source.content or "", + "content": source.tool_schema or "", "chat_time": source.chat_time, "message_id": source.message_id, } @@ -74,7 +82,7 @@ def parse_fast( if isinstance(content, dict): content = content["text"] - # Extract tool_schema content and remaining content + # Replace tool_schema content with "omitted" in remaining content content_wo_tool_schema = re.sub( r"(.*?)", r"omitted", @@ -87,7 +95,7 @@ def parse_fast( TextualMemoryItem( memory=content_wo_tool_schema, metadata=TreeNodeTextualMemoryMetadata( - memory_type="LongTermMemory", + memory_type="LongTermMemory", # only choce long term memory for system messages as a placeholder status="activated", tags=["mode:fast"], sources=[source], @@ -106,8 +114,12 @@ def parse_fine( content = content["text"] try: tool_schema = json.loads(content) - assert isinstance(tool_schema, list), "Tool schema must be a list" + assert isinstance(tool_schema, list), "Tool schema must be a list[dict]" except json.JSONDecodeError: + logger.warning(f"[SystemParser] Failed to parse tool schema: {content}") + return [] + except AssertionError: + logger.warning(f"[SystemParser] Tool schema must be a list[dict]: {content}") return [] return [ @@ -115,7 +127,7 @@ def parse_fine( id=str(uuid.uuid4()), memory=json.dumps(tool_schema), metadata=TreeNodeTextualMemoryMetadata( - memory_type="tool_schema", + memory_type="ToolSchemaMemory", ), ) ] diff --git a/src/memos/mem_reader/read_multi_modal/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py index 8d42b69a7..8d5ac7c24 100644 --- a/src/memos/mem_reader/read_multi_modal/tool_parser.py +++ b/src/memos/mem_reader/read_multi_modal/tool_parser.py @@ -56,7 +56,18 @@ def create_source( for part in raw_content: if isinstance(part, dict): part_type = part.get("type", "") - if part_type == "file": + if part_type == "text": + sources.append( + SourceMessage( + type="text", + role=role, + chat_time=chat_time, + message_id=message_id, + content=part.get("text", ""), + tool_call_id=tool_call_id, + ) + ) + elif part_type == "file": file_info = part.get("file", {}) sources.append( SourceMessage( @@ -64,45 +75,59 @@ def create_source( role=role, chat_time=chat_time, message_id=message_id, - doc_path=file_info.get("filename") or file_info.get("file_id", ""), content=file_info.get("file_data", ""), + filename=file_info.get("filename", ""), + file_id=file_info.get("file_id", ""), tool_call_id=tool_call_id, original_part=part, ) ) - else: - # image_url, input_audio, etc. + elif part_type == "image_url": + file_info = part.get("image_url", {}) sources.append( SourceMessage( - type=part_type, + type="image_url", role=role, chat_time=chat_time, message_id=message_id, - content=f"[{part_type}]", + content=file_info.get("url", ""), + detail=file_info.get("detail", "auto"), tool_call_id=tool_call_id, original_part=part, ) ) + elif part_type == "input_audio": + file_info = part.get("input_audio", {}) + sources.append( + SourceMessage( + type="input_audio", + role=role, + chat_time=chat_time, + message_id=message_id, + content=file_info.get("data", ""), + format=file_info.get("format", "wav"), + tool_call_id=tool_call_id, + original_part=part, + ) + ) + else: + logger.warning(f"[ToolParser] Unsupported part type: {part_type}") + continue else: # Simple string content message: single SourceMessage - content = raw_content - if content: + if raw_content: sources.append( SourceMessage( type="chat", role=role, chat_time=chat_time, message_id=message_id, - content=content, + content=raw_content, tool_call_id=tool_call_id, ) ) - return ( - sources - if len(sources) > 1 - else (sources[0] if sources else SourceMessage(type="chat", role=role)) - ) + return sources def rebuild_from_source( self, @@ -117,6 +142,7 @@ def rebuild_from_source( if isinstance(original, dict) and "type" in original: return { "role": source.role or "user", + "tool_call_id": source.tool_call_id or "", "content": [original], "chat_time": source.chat_time, "message_id": source.message_id, @@ -126,14 +152,27 @@ def rebuild_from_source( return original # Priority 2: Rebuild from source fields - if source.type == "file": + if source.type == "text": return { - "role": source.role or "user", + "role": source.role or "tool", + "content": [ + { + "type": "text", + "text": source.content or "", + } + ], + "chat_time": source.chat_time, + "message_id": source.message_id, + } + elif source.type == "file": + return { + "role": source.role or "tool", "content": [ { "type": "file", "file": { - "filename": source.doc_path or "", + "file_id": source.file_id or "", + "filename": source.filename or "", "file_data": source.content or "", }, } @@ -141,12 +180,42 @@ def rebuild_from_source( "chat_time": source.chat_time, "message_id": source.message_id, } + elif source.type == "image_url": + return { + "role": source.role or "tool", + "content": [ + { + "type": "image_url", + "image_url": { + "url": source.content or "", + "detail": source.detail or "auto", + }, + } + ], + "chat_time": source.chat_time, + "message_id": source.message_id, + } + elif source.type == "input_audio": + return { + "role": source.role or "tool", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": source.content or "", + "format": source.format or "wav", + }, + } + ], + "chat_time": source.chat_time, + "message_id": source.message_id, + } # Simple text message return { "role": "tool", "content": source.content or "", - "tool_call_id": source.message_id or "", # tool_call_id might be in message_id + "tool_call_id": source.message_id or "", "chat_time": source.chat_time, "message_id": source.message_id, } @@ -157,11 +226,30 @@ def parse_fast( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - memory = json.dumps(message) - source = self.create_source(message, info) + role = message.get("role", "") + content = message.get("content", "") + chat_time = message.get("chat_time", None) + + if role != "user": + logger.warning(f"[ToolParser] Expected role is `user`, got {role}") + return [] + parts = [f"{role}: "] + if chat_time: + parts.append(f"[{chat_time}]: ") + prefix = "".join(parts) + content = json.dumps(content) if isinstance(content, list) else content + line = f"{prefix}{content}\n" + if not line: + return [] + memory_type = ( + "LongTermMemory" # only choce long term memory for tool messages as a placeholder + ) + + sources = self.create_source(message, info) return [ TextualMemoryItem( - memory=memory, metadata=TreeNodeTextualMemoryMetadata(sources=[source]) + memory=line, + metadata=TreeNodeTextualMemoryMetadata(memory_type=memory_type, sources=sources), ) ] @@ -171,4 +259,25 @@ def parse_fine( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: + content = message.get("content", "") + if isinstance(content, list): + part_type = content[0].get("type", "") + if part_type == "text": + # text will fine parse in full chat content, no need to parse specially + return [] + elif part_type == "file": + # TODO: use OCR to extract text from file and generate mem by llm + content = content[0].get("file", {}).get("file_data", "") + elif part_type == "image_url": + # TODO: use multi-modal llm to generate mem by image url + content = content[0].get("image_url", {}).get("url", "") + elif part_type == "input_audio": + # TODO: unsupport audio for now + return [] + else: + logger.warning(f"[ToolParser] Unsupported part type: {part_type}") + return [] + else: + # simple string content message, fine parse in full chat content, no need to parse specially + return [] return [] diff --git a/src/memos/mem_reader/simple_struct.py b/src/memos/mem_reader/simple_struct.py index 53a7de035..7f7b16234 100644 --- a/src/memos/mem_reader/simple_struct.py +++ b/src/memos/mem_reader/simple_struct.py @@ -223,6 +223,7 @@ def _make_memory_item( background: str = "", type_: str = "fact", confidence: float = 0.99, + **kwargs, ) -> TextualMemoryItem: """construct memory item""" info_ = info.copy() @@ -245,6 +246,7 @@ def _make_memory_item( confidence=confidence, type=type_, info=info_, + **kwargs, ), ) diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 76b252f54..75a16bace 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -24,7 +24,7 @@ class SourceMessage(BaseModel): - type: Source kind (e.g., "chat", "doc", "web", "file", "system", ...). If not provided, upstream logic may infer it: presence of `role` ⇒ "chat"; otherwise ⇒ "doc". - - role: Conversation role ("user" | "assistant" | "system") when the + - role: Conversation role ("user" | "assistant" | "system" | "tool") when the source is a chat turn. - content: Minimal reproducible snippet from the source. If omitted, upstream may fall back to `doc_path` / `url` / `message_id`. @@ -35,7 +35,7 @@ class SourceMessage(BaseModel): """ type: str | None = "chat" - role: Literal["user", "assistant", "system"] | None = None + role: Literal["user", "assistant", "system", "tool"] | None = None chat_time: str | None = None message_id: str | None = None content: str | None = None @@ -104,8 +104,8 @@ class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata): "LongTermMemory", "UserMemory", "OuterMemory", - "ToolSchema", - "ToolTrajectory", + "ToolSchemaMemory", + "ToolTrajectoryMemory", ] = Field(default="WorkingMemory", description="Memory lifecycle type.") sources: list[SourceMessage] | None = Field( default=None, description="Multiple origins of the memory (e.g., URLs, notes)." diff --git a/src/memos/templates/tool_mem_prompts.py b/src/memos/templates/tool_mem_prompts.py new file mode 100644 index 000000000..7d5363956 --- /dev/null +++ b/src/memos/templates/tool_mem_prompts.py @@ -0,0 +1,84 @@ +TOOL_TRAJECTORY_PROMPT_ZH = """ +你是一个专业的工具调用轨迹提取专家。你的任务是从给定的对话消息中提取完整的工具调用轨迹经验。 + +## 提取规则： +1. 只有当对话中存在有价值的工具调用过程时才进行提取 +2. 有价值的轨迹至少包含以下元素： + - 用户的问题（user message） + - 助手的工具调用尝试（assistant message with tool_calls） + - 工具的执行结果（tool message with tool_call_id and content，无论成功或失败） + - 助手的响应（assistant message，无论是否给出最终答案） + +## 输出格式： +返回一个JSON数组，格式如下： +```json +[ + { + "trajectory": "自然语言输出包含'任务、使用的工具、工具观察、最终回答'的完整精炼的总结，体现顺序", + "tool_used_status": [ + { + "used_tool": "工具名1", + "success_rate": "0.0-1.0之间的数值，表示该工具在本次轨迹中的成功率", + "error_type": "调用失败时的错误类型和描述，成功时为空字符串", + "experience": "该工具的使用经验，比如常见的参数模式、执行特点、结果解读方式等" + } + ] + } +] +``` + +## 注意事项： +- 如果对话中没有完整的工具调用轨迹，返回空数组 +- 每个轨迹必须是独立的完整过程 +- 一个轨迹中可能涉及多个工具的使用，每个工具在tool_used_status中独立记录 +- 只提取事实内容，不要添加任何解释或额外信息 +- 确保返回的是有效的JSON格式 + +请分析以下对话消息并提取工具调用轨迹： + +{messages} + +""" + + +TOOL_TRAJECTORY_PROMPT_EN = """ +You are a professional tool call trajectory extraction expert. Your task is to extract valuable tool call trajectory experiences from given conversation messages. + +## Extraction Rules: +1. Only extract when there are valuable tool calling processes in the conversation +2. Valuable trajectories must contain at least the following elements: + - User's question (user message) + - Assistant's tool call attempt (assistant message with tool_calls) + - Tool execution results (tool message with tool_call_id and content, regardless of success or failure) + - Assistant's response (assistant message, whether or not a final answer is given) + +## Output Format: +Return a JSON array in the following format: +```json +[ + { + "trajectory": "Natural language summary containing 'task, tools used, tool observations, final answer' in a complete and refined manner, reflecting the sequence", + "tool_used_status": [ + { + "used_tool": "Tool Name 1", + "success_rate": "Numerical value between 0.0-1.0, indicating the success rate of this tool in the current trajectory", + "error_type": "Error type and description when call fails, empty string when successful", + "experience": "Usage experience of this tool, such as common parameter patterns, execution characteristics, result interpretation methods, etc." + } + ] + } +] +``` + +## Notes: +- If there are no complete tool call trajectories in the conversation, return an empty array +- Each trajectory must be an independent complete process +- Multiple tools may be used in one trajectory, each tool is recorded independently in tool_used_status +- Only extract factual content, do not add any additional explanations or information +- Ensure the returned content is valid JSON format + +Please analyze the following conversation messages and extract tool call trajectories: + +{messages} + +""" From b6efc875e05d7b50ae24e10e41ee0c1dd108d891 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 2 Dec 2025 18:58:22 +0800 Subject: [PATCH 06/13] modify multi-modal code --- .../mem_reader/read_multi_modal/system_parser.py | 15 ++++++++++----- .../mem_reader/read_multi_modal/tool_parser.py | 16 +++++++++------- .../textual/tree_text_memory/organize/manager.py | 7 ++++++- .../chat_completion_assistant_message_param.py | 7 ++++--- .../chat_completion_system_message_param.py | 5 +++-- .../chat_completion_tool_message_param.py | 3 +-- .../chat_completion_user_message_param.py | 3 +-- 7 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/memos/mem_reader/read_multi_modal/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py index 9056a050c..e08279dc1 100644 --- a/src/memos/mem_reader/read_multi_modal/system_parser.py +++ b/src/memos/mem_reader/read_multi_modal/system_parser.py @@ -37,17 +37,21 @@ def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None): def create_source( self, - message: str, + message: ChatCompletionSystemMessageParam, info: dict[str, Any], ) -> SourceMessage: """Create SourceMessage from system message.""" + content = message["content"] + if isinstance(content, dict): + content = content["text"] + content_wo_tool_schema = re.sub( r"(.*?)", r"omitted", - message, + content, flags=re.DOTALL, ) - tool_schema_match = re.search(r"(.*?)", message, re.DOTALL) + tool_schema_match = re.search(r"(.*?)", content, re.DOTALL) tool_schema_content = tool_schema_match.group(1) if tool_schema_match else "" return SourceMessage( @@ -90,7 +94,7 @@ def parse_fast( flags=re.DOTALL, ) - source = self.create_source(content, info) + source = self.create_source(message, info) return [ TextualMemoryItem( memory=content_wo_tool_schema, @@ -125,9 +129,10 @@ def parse_fine( return [ TextualMemoryItem( id=str(uuid.uuid4()), - memory=json.dumps(tool_schema), + memory=json.dumps(schema), metadata=TreeNodeTextualMemoryMetadata( memory_type="ToolSchemaMemory", ), ) + for schema in tool_schema ] diff --git a/src/memos/mem_reader/read_multi_modal/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py index 8d5ac7c24..72937cf9a 100644 --- a/src/memos/mem_reader/read_multi_modal/tool_parser.py +++ b/src/memos/mem_reader/read_multi_modal/tool_parser.py @@ -230,26 +230,28 @@ def parse_fast( content = message.get("content", "") chat_time = message.get("chat_time", None) - if role != "user": - logger.warning(f"[ToolParser] Expected role is `user`, got {role}") + if role != "tool": + logger.warning(f"[ToolParser] Expected role is `tool`, got {role}") return [] parts = [f"{role}: "] if chat_time: parts.append(f"[{chat_time}]: ") prefix = "".join(parts) - content = json.dumps(content) if isinstance(content, list) else content + content = json.dumps(content) if isinstance(content, list | dict) else content line = f"{prefix}{content}\n" if not line: return [] - memory_type = ( - "LongTermMemory" # only choce long term memory for tool messages as a placeholder - ) sources = self.create_source(message, info) return [ TextualMemoryItem( memory=line, - metadata=TreeNodeTextualMemoryMetadata(memory_type=memory_type, sources=sources), + metadata=TreeNodeTextualMemoryMetadata( + memory_type="LongTermMemory", # only choce long term memory for tool messages as a placeholder + status="activated", + tags=["mode:fast"], + sources=sources, + ), ) ] diff --git a/src/memos/memories/textual/tree_text_memory/organize/manager.py b/src/memos/memories/textual/tree_text_memory/organize/manager.py index a71fee02f..31cf89ae1 100644 --- a/src/memos/memories/textual/tree_text_memory/organize/manager.py +++ b/src/memos/memories/textual/tree_text_memory/organize/manager.py @@ -186,7 +186,12 @@ def _process_memory(self, memory: TextualMemoryItem, user_name: str | None = Non ) futures.append(("working", f_working)) - if memory.metadata.memory_type in ("LongTermMemory", "UserMemory"): + if memory.metadata.memory_type in ( + "LongTermMemory", + "UserMemory", + "ToolSchemaMemory", + "ToolTrajectoryMemory", + ): f_graph = ex.submit( self._add_to_graph_memory, memory=memory, diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_assistant_message_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_assistant_message_param.py index a742de3a9..3c5638788 100644 --- a/src/memos/types/openai_chat_completion_types/chat_completion_assistant_message_param.py +++ b/src/memos/types/openai_chat_completion_types/chat_completion_assistant_message_param.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import Literal, TypeAlias from typing_extensions import Required, TypedDict @@ -35,7 +34,7 @@ class ChatCompletionAssistantMessageParam(TypedDict, total=False): [Learn more](https://platform.openai.com/docs/guides/audio). """ - content: str | Iterable[ContentArrayOfContentPart] | None + content: str | list[ContentArrayOfContentPart] | ContentArrayOfContentPart | None """The contents of the assistant message. Required unless `tool_calls` or `function_call` is specified. @@ -44,7 +43,9 @@ class ChatCompletionAssistantMessageParam(TypedDict, total=False): refusal: str | None """The refusal message by the assistant.""" - tool_calls: Iterable[ChatCompletionMessageToolCallUnionParam] + tool_calls: ( + list[ChatCompletionMessageToolCallUnionParam] | ChatCompletionMessageToolCallUnionParam + ) """The tool calls generated by the model, such as function calls.""" chat_time: str | None diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_system_message_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_system_message_param.py index 7faa90e2e..ea2101229 100644 --- a/src/memos/types/openai_chat_completion_types/chat_completion_system_message_param.py +++ b/src/memos/types/openai_chat_completion_types/chat_completion_system_message_param.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import Literal from typing_extensions import Required, TypedDict @@ -14,7 +13,9 @@ class ChatCompletionSystemMessageParam(TypedDict, total=False): - content: Required[str | Iterable[ChatCompletionContentPartTextParam]] + content: Required[ + str | list[ChatCompletionContentPartTextParam] | ChatCompletionContentPartTextParam + ] """The contents of the system message.""" role: Required[Literal["system"]] diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_tool_message_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_tool_message_param.py index c03220915..99c845d11 100644 --- a/src/memos/types/openai_chat_completion_types/chat_completion_tool_message_param.py +++ b/src/memos/types/openai_chat_completion_types/chat_completion_tool_message_param.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import Literal from typing_extensions import Required, TypedDict @@ -14,7 +13,7 @@ class ChatCompletionToolMessageParam(TypedDict, total=False): - content: Required[str | Iterable[ChatCompletionContentPartParam]] + content: Required[str | list[ChatCompletionContentPartParam] | ChatCompletionContentPartParam] """The contents of the tool message.""" role: Required[Literal["tool"]] diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_user_message_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_user_message_param.py index 2c2a1f23f..8c004f340 100644 --- a/src/memos/types/openai_chat_completion_types/chat_completion_user_message_param.py +++ b/src/memos/types/openai_chat_completion_types/chat_completion_user_message_param.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import Literal from typing_extensions import Required, TypedDict @@ -14,7 +13,7 @@ class ChatCompletionUserMessageParam(TypedDict, total=False): - content: Required[str | Iterable[ChatCompletionContentPartParam]] + content: Required[str | list[ChatCompletionContentPartParam] | ChatCompletionContentPartParam] """The contents of the user message.""" role: Required[Literal["user"]] From 492571bf8a78e010ae51afacad074830a064f19a Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 2 Dec 2025 20:43:13 +0800 Subject: [PATCH 07/13] pref support multi-modal messages --- .../textual/prefer_text_memory/spliter.py | 2 +- .../textual/prefer_text_memory/utils.py | 33 ++++++++++++++++--- src/memos/multi_mem_cube/single_cube.py | 7 ++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/memos/memories/textual/prefer_text_memory/spliter.py b/src/memos/memories/textual/prefer_text_memory/spliter.py index 3059d611b..a54036778 100644 --- a/src/memos/memories/textual/prefer_text_memory/spliter.py +++ b/src/memos/memories/textual/prefer_text_memory/spliter.py @@ -87,7 +87,7 @@ def _split_with_overlap(self, data: MessageList) -> list[MessageList]: # overlap 1 turns (Q + A = 2) context = copy.deepcopy(chunk[-2:]) if i + 1 < len(data) else [] chunk = context - if chunk and len(chunk) % 2 == 0: + if chunk: chunks.append(chunk) return chunks diff --git a/src/memos/memories/textual/prefer_text_memory/utils.py b/src/memos/memories/textual/prefer_text_memory/utils.py index 76d4b4211..03d2ef923 100644 --- a/src/memos/memories/textual/prefer_text_memory/utils.py +++ b/src/memos/memories/textual/prefer_text_memory/utils.py @@ -1,3 +1,4 @@ +import json import re from memos.dependency import require_python_package @@ -9,12 +10,36 @@ def convert_messages_to_string(messages: MessageList) -> str: """Convert a list of messages to a string.""" message_text = "" for message in messages: + content = message.get("content", "") + content = ( + content.strip() + if isinstance(content, str) + else json.dumps(content, ensure_ascii=False).strip() + ) + if message["role"] == "system": + continue if message["role"] == "user": - message_text += f"Query: {message['content']}\n" if message["content"].strip() else "" + message_text += f"User: {content}\n" if content else "" elif message["role"] == "assistant": - message_text += f"Answer: {message['content']}\n" if message["content"].strip() else "" - message_text = message_text.strip() - return message_text + tool_calls = message.get("tool_calls", []) + tool_calls_str = ( + f"[tool_calls]: {json.dumps(tool_calls, ensure_ascii=False)}" if tool_calls else "" + ) + line_str = ( + f"Assistant: {content} {tool_calls_str}".strip() + if content or tool_calls_str + else "" + ) + message_text += f"{line_str}\n" if line_str else "" + elif message["role"] == "tool": + tool_call_id = message.get("tool_call_id", "") + line_str = ( + f"Tool: {content} [tool_call_id]: {tool_call_id}".strip() + if tool_call_id + else f"Tool: {content}".strip() + ) + message_text += f"{line_str}\n" if line_str else "" + return message_text.strip() @require_python_package( diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index 880646939..6c1ff00c0 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -495,6 +495,13 @@ def _process_pref_mem( if os.getenv("ENABLE_PREFERENCE_MEMORY", "false").lower() != "true": return [] + if add_req.messages is None or isinstance(add_req.messages, str): + return [] + + for message in add_req.messages: + if message.get("role", None) is None: + return [] + target_session_id = add_req.session_id or "default_session" if sync_mode == "async": From 89f11aa524710e3a88029fe1359edfbad30e3eca Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 2 Dec 2025 20:58:24 +0800 Subject: [PATCH 08/13] modify bug in chat handle --- src/memos/api/handlers/chat_handler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 1054644d2..968b9f5e6 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -142,7 +142,9 @@ def handle_chat_complete(self, chat_req: APIChatCompleteRequest) -> dict[str, An # Step 2: Build system prompt system_prompt = self._build_system_prompt( - filtered_memories, search_response.data["pref_string"], chat_req.system_prompt + filtered_memories, + search_response.data.get("pref_string", ""), + chat_req.system_prompt, ) # Prepare message history @@ -257,7 +259,7 @@ def generate_chat_response() -> Generator[str, None, None]: # Step 2: Build system prompt with memories system_prompt = self._build_system_prompt( filtered_memories, - search_response.data["pref_string"], + search_response.data.get("pref_string", ""), chat_req.system_prompt, ) @@ -436,7 +438,7 @@ def generate_chat_response() -> Generator[str, None, None]: # Step 2: Build system prompt with memories system_prompt = self._build_enhance_system_prompt( - filtered_memories, search_response.data["pref_string"] + filtered_memories, search_response.data.get("pref_string", "") ) # Prepare messages From 152844a996229ca25ec45c5aa5acd6a9a0b7d58d Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 2 Dec 2025 21:14:27 +0800 Subject: [PATCH 09/13] fix pre commit --- poetry.lock | 42 ++++++++++---------- src/memos/api/product_models.py | 2 +- src/memos/mem_scheduler/general_scheduler.py | 4 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index 940697b1c..c6c82cdbb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "absl-py" @@ -54,7 +54,7 @@ description = "Timeout context manager for asyncio programs" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "(python_version == \"3.10\" or python_version == \"3.11\") and python_full_version < \"3.11.3\" and (extra == \"mem-scheduler\" or extra == \"all\")" +markers = "python_full_version < \"3.11.3\" and (extra == \"mem-scheduler\" or extra == \"all\")" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -1080,7 +1080,7 @@ description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.9" groups = ["main", "eval"] -markers = "(python_version == \"3.10\" or python_version == \"3.11\" or python_version == \"3.12\" or python_version == \"3.13\") and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" +markers = "(platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\") and python_version < \"3.14\"" files = [ {file = "greenlet-3.2.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:1afd685acd5597349ee6d7a88a8bec83ce13c106ac78c196ee9dde7c04fe87be"}, {file = "greenlet-3.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:761917cac215c61e9dc7324b2606107b3b292a8349bdebb31503ab4de3f559ac"}, @@ -2623,7 +2623,7 @@ files = [ {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668"}, {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-win_amd64.whl", hash = "sha256:9e4fa264f4d8a4eb0cdbd34beadc029f453b3bafae02401e999cf3d5a5af75f8"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-cuda-cupti-cu12" @@ -2639,7 +2639,7 @@ files = [ {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73"}, {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-win_amd64.whl", hash = "sha256:bbe6ae76e83ce5251b56e8c8e61a964f757175682bbad058b170b136266ab00a"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-cuda-nvrtc-cu12" @@ -2653,7 +2653,7 @@ files = [ {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53"}, {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:f7007dbd914c56bd80ea31bc43e8e149da38f68158f423ba845fc3292684e45a"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-cuda-runtime-cu12" @@ -2669,7 +2669,7 @@ files = [ {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8"}, {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:86c58044c824bf3c173c49a2dbc7a6c8b53cb4e4dca50068be0bf64e9dab3f7f"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-cudnn-cu12" @@ -2683,7 +2683,7 @@ files = [ {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2"}, {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-win_amd64.whl", hash = "sha256:d7af0f8a4f3b4b9dbb3122f2ef553b45694ed9c384d5a75bab197b8eefb79ab8"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [package.dependencies] nvidia-cublas-cu12 = "*" @@ -2702,7 +2702,7 @@ files = [ {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca"}, {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-win_amd64.whl", hash = "sha256:6048ebddfb90d09d2707efb1fd78d4e3a77cb3ae4dc60e19aab6be0ece2ae464"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [package.dependencies] nvidia-nvjitlink-cu12 = "*" @@ -2718,7 +2718,7 @@ files = [ {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-curand-cu12" @@ -2734,7 +2734,7 @@ files = [ {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7b2ed8e95595c3591d984ea3603dd66fe6ce6812b886d59049988a712ed06b6e"}, {file = "nvidia_curand_cu12-10.3.7.77-py3-none-win_amd64.whl", hash = "sha256:6d6d935ffba0f3d439b7cd968192ff068fafd9018dbf1b85b37261b13cfc9905"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-cusolver-cu12" @@ -2750,7 +2750,7 @@ files = [ {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dbbe4fc38ec1289c7e5230e16248365e375c3673c9c8bac5796e2e20db07f56e"}, {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-win_amd64.whl", hash = "sha256:6813f9d8073f555444a8705f3ab0296d3e1cb37a16d694c5fc8b862a0d8706d7"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [package.dependencies] nvidia-cublas-cu12 = "*" @@ -2771,7 +2771,7 @@ files = [ {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f"}, {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-win_amd64.whl", hash = "sha256:4acb8c08855a26d737398cba8fb6f8f5045d93f82612b4cfd84645a2332ccf20"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [package.dependencies] nvidia-nvjitlink-cu12 = "*" @@ -2788,7 +2788,7 @@ files = [ {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46"}, {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-win_amd64.whl", hash = "sha256:3b325bcbd9b754ba43df5a311488fca11a6b5dc3d11df4d190c000cf1a0765c7"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-nccl-cu12" @@ -2801,7 +2801,7 @@ files = [ {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-nvjitlink-cu12" @@ -2815,7 +2815,7 @@ files = [ {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41"}, {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-win_amd64.whl", hash = "sha256:e61120e52ed675747825cdd16febc6a0730537451d867ee58bee3853b1b13d1c"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "nvidia-nvtx-cu12" @@ -2831,7 +2831,7 @@ files = [ {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1"}, {file = "nvidia_nvtx_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:2fb11a4af04a5e6c84073e6404d26588a34afd35379f0855a99797897efa75c0"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [[package]] name = "ollama" @@ -3919,7 +3919,7 @@ files = [ {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, ] -markers = {main = "extra == \"all\" and platform_system == \"Windows\" or sys_platform == \"win32\"", eval = "platform_system == \"Windows\""} +markers = {main = "platform_system == \"Windows\" and extra == \"all\" or sys_platform == \"win32\"", eval = "platform_system == \"Windows\""} [[package]] name = "pyyaml" @@ -4917,7 +4917,7 @@ files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] -markers = {main = "extra == \"all\" or extra == \"pref-mem\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\""} +markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and (extra == \"all\" or extra == \"pref-mem\") or extra == \"pref-mem\" or extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\" or python_version >= \"3.12\""} [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] @@ -5408,7 +5408,7 @@ files = [ {file = "triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42"}, {file = "triton-3.3.1-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6139aeb04a146b0b8e0fbbd89ad1e65861c57cfed881f21d62d3cb94a36bab7"}, ] -markers = {main = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"all\"", eval = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +markers = {main = "platform_machine == \"x86_64\" and extra == \"all\" and platform_system == \"Linux\"", eval = "platform_machine == \"x86_64\" and platform_system == \"Linux\""} [package.dependencies] setuptools = ">=40.8.0" @@ -5617,7 +5617,7 @@ description = "Fast implementation of asyncio event loop on top of libuv" optional = false python-versions = ">=3.8.0" groups = ["main"] -markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"" +markers = "platform_python_implementation != \"PyPy\" and sys_platform != \"win32\" and sys_platform != \"cygwin\"" files = [ {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f"}, {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d"}, diff --git a/src/memos/api/product_models.py b/src/memos/api/product_models.py index 16ae86638..ffe736aa3 100644 --- a/src/memos/api/product_models.py +++ b/src/memos/api/product_models.py @@ -6,7 +6,7 @@ # Import message types from core types module from memos.log import get_logger -from memos.types import MessageDict, MessageList, MessagesType, PermissionDict, SearchMode +from memos.types import MessageDict, PermissionDict, SearchMode logger = get_logger(__name__) diff --git a/src/memos/mem_scheduler/general_scheduler.py b/src/memos/mem_scheduler/general_scheduler.py index 601c935a2..2145dee9f 100644 --- a/src/memos/mem_scheduler/general_scheduler.py +++ b/src/memos/mem_scheduler/general_scheduler.py @@ -309,7 +309,9 @@ def log_add_messages(self, msg: ScheduleMessageItem): if missing_ids: content_preview = ( - msg.content[:200] + "..." if isinstance(msg.content, str) and len(msg.content) > 200 else msg.content + msg.content[:200] + "..." + if isinstance(msg.content, str) and len(msg.content) > 200 + else msg.content ) logger.warning( "Missing TextualMemoryItem(s) during add log preparation. " From 2e68bac864446de8abfb5e77c24886d10c0828a3 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 3 Dec 2025 10:48:37 +0800 Subject: [PATCH 10/13] modify code --- src/memos/mem_reader/read_multi_modal/tool_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memos/mem_reader/read_multi_modal/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py index 72937cf9a..a3d9e6b91 100644 --- a/src/memos/mem_reader/read_multi_modal/tool_parser.py +++ b/src/memos/mem_reader/read_multi_modal/tool_parser.py @@ -268,11 +268,12 @@ def parse_fine( # text will fine parse in full chat content, no need to parse specially return [] elif part_type == "file": - # TODO: use OCR to extract text from file and generate mem by llm - content = content[0].get("file", {}).get("file_data", "") + # use file content parser to parse file content, no need to parse here + return [] elif part_type == "image_url": # TODO: use multi-modal llm to generate mem by image url content = content[0].get("image_url", {}).get("url", "") + return [] elif part_type == "input_audio": # TODO: unsupport audio for now return [] @@ -282,4 +283,3 @@ def parse_fine( else: # simple string content message, fine parse in full chat content, no need to parse specially return [] - return [] From a04b5f5fe83724eb3900d7069f21b45b4f50f6ef Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 3 Dec 2025 15:56:37 +0800 Subject: [PATCH 11/13] add tool search --- src/memos/api/handlers/formatters_handler.py | 34 ++++ src/memos/api/product_models.py | 15 ++ .../read_multi_modal/tool_parser.py | 109 +----------- .../mem_scheduler/optimized_scheduler.py | 4 + src/memos/memories/textual/tree.py | 4 + .../tree_text_memory/retrieve/recall.py | 8 +- .../tree_text_memory/retrieve/searcher.py | 160 +++++++++++++++++- src/memos/multi_mem_cube/single_cube.py | 18 +- 8 files changed, 236 insertions(+), 116 deletions(-) diff --git a/src/memos/api/handlers/formatters_handler.py b/src/memos/api/handlers/formatters_handler.py index 976be87bb..88875cacc 100644 --- a/src/memos/api/handlers/formatters_handler.py +++ b/src/memos/api/handlers/formatters_handler.py @@ -90,3 +90,37 @@ def post_process_pref_mem( memories_result["pref_note"] = pref_note return memories_result + + +def post_process_textual_mem( + memories_result: dict[str, Any], + text_formatted_mem: list[dict[str, Any]], + mem_cube_id: str, +) -> dict[str, Any]: + """ + Post-process text and tool memory results. + """ + fact_mem = [ + mem + for mem in text_formatted_mem + if mem["metadata"]["memory_type"] not in ["ToolSchemaMemory", "ToolTrajectoryMemory"] + ] + tool_mem = [ + mem + for mem in text_formatted_mem + if mem["metadata"]["memory_type"] in ["ToolSchemaMemory", "ToolTrajectoryMemory"] + ] + + memories_result["text_mem"].append( + { + "cube_id": mem_cube_id, + "memories": fact_mem, + } + ) + memories_result["tool_mem"].append( + { + "cube_id": mem_cube_id, + "memories": tool_mem, + } + ) + return memories_result diff --git a/src/memos/api/product_models.py b/src/memos/api/product_models.py index ffe736aa3..acdb773e9 100644 --- a/src/memos/api/product_models.py +++ b/src/memos/api/product_models.py @@ -326,6 +326,21 @@ class APISearchRequest(BaseRequest): ), ) + search_tool_memory: bool = Field( + True, + description=( + "Whether to retrieve tool memories along with general memories. " + "If enabled, the system will automatically recall tool memories " + "relevant to the query. Default: True." + ), + ) + + tool_mem_top_k: int = Field( + 6, + ge=0, + description="Number of tool memories to retrieve (top-K). Default: 6.", + ) + # ==== Filter conditions ==== # TODO: maybe add detailed description later filter: dict[str, Any] | None = Field( diff --git a/src/memos/mem_reader/read_multi_modal/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py index a3d9e6b91..cdd3620cb 100644 --- a/src/memos/mem_reader/read_multi_modal/tool_parser.py +++ b/src/memos/mem_reader/read_multi_modal/tool_parser.py @@ -135,91 +135,6 @@ def rebuild_from_source( ) -> ChatCompletionToolMessageParam: """Rebuild tool message from SourceMessage.""" - # Priority 1: Use original_part if available - if hasattr(source, "original_part") and source.original_part: - original = source.original_part - # If it's a content part, wrap it in a message - if isinstance(original, dict) and "type" in original: - return { - "role": source.role or "user", - "tool_call_id": source.tool_call_id or "", - "content": [original], - "chat_time": source.chat_time, - "message_id": source.message_id, - } - # If it's already a full message, return it - if isinstance(original, dict) and "role" in original: - return original - - # Priority 2: Rebuild from source fields - if source.type == "text": - return { - "role": source.role or "tool", - "content": [ - { - "type": "text", - "text": source.content or "", - } - ], - "chat_time": source.chat_time, - "message_id": source.message_id, - } - elif source.type == "file": - return { - "role": source.role or "tool", - "content": [ - { - "type": "file", - "file": { - "file_id": source.file_id or "", - "filename": source.filename or "", - "file_data": source.content or "", - }, - } - ], - "chat_time": source.chat_time, - "message_id": source.message_id, - } - elif source.type == "image_url": - return { - "role": source.role or "tool", - "content": [ - { - "type": "image_url", - "image_url": { - "url": source.content or "", - "detail": source.detail or "auto", - }, - } - ], - "chat_time": source.chat_time, - "message_id": source.message_id, - } - elif source.type == "input_audio": - return { - "role": source.role or "tool", - "content": [ - { - "type": "input_audio", - "input_audio": { - "data": source.content or "", - "format": source.format or "wav", - }, - } - ], - "chat_time": source.chat_time, - "message_id": source.message_id, - } - - # Simple text message - return { - "role": "tool", - "content": source.content or "", - "tool_call_id": source.message_id or "", - "chat_time": source.chat_time, - "message_id": source.message_id, - } - def parse_fast( self, message: ChatCompletionToolMessageParam, @@ -261,25 +176,5 @@ def parse_fine( info: dict[str, Any], **kwargs, ) -> list[TextualMemoryItem]: - content = message.get("content", "") - if isinstance(content, list): - part_type = content[0].get("type", "") - if part_type == "text": - # text will fine parse in full chat content, no need to parse specially - return [] - elif part_type == "file": - # use file content parser to parse file content, no need to parse here - return [] - elif part_type == "image_url": - # TODO: use multi-modal llm to generate mem by image url - content = content[0].get("image_url", {}).get("url", "") - return [] - elif part_type == "input_audio": - # TODO: unsupport audio for now - return [] - else: - logger.warning(f"[ToolParser] Unsupported part type: {part_type}") - return [] - else: - # simple string content message, fine parse in full chat content, no need to parse specially - return [] + # tool message no special multimodal handling is required in fine mode. + return [] diff --git a/src/memos/mem_scheduler/optimized_scheduler.py b/src/memos/mem_scheduler/optimized_scheduler.py index a85c533a0..f99360a86 100644 --- a/src/memos/mem_scheduler/optimized_scheduler.py +++ b/src/memos/mem_scheduler/optimized_scheduler.py @@ -159,6 +159,8 @@ def mix_search_memories( search_filter=search_filter, search_priority=search_priority, info=info, + search_tool_memory=search_req.search_tool_memory, + tool_mem_top_k=search_req.tool_mem_top_k, ) # Try to get pre-computed memories if available @@ -182,6 +184,8 @@ def mix_search_memories( top_k=search_req.top_k, user_name=user_context.mem_cube_id, info=info, + search_tool_memory=search_req.search_tool_memory, + tool_mem_top_k=search_req.tool_mem_top_k, ) memories = merged_memories[: search_req.top_k] diff --git a/src/memos/memories/textual/tree.py b/src/memos/memories/textual/tree.py index ad2bcd9c4..cad850d2d 100644 --- a/src/memos/memories/textual/tree.py +++ b/src/memos/memories/textual/tree.py @@ -166,6 +166,8 @@ def search( search_priority: dict | None = None, search_filter: dict | None = None, user_name: str | None = None, + search_tool_memory: bool = False, + tool_mem_top_k: int = 6, **kwargs, ) -> list[TextualMemoryItem]: """Search for memories based on a query. @@ -223,6 +225,8 @@ def search( search_priority, user_name=user_name, plugin=kwargs.get("plugin", False), + search_tool_memory=search_tool_memory, + tool_mem_top_k=tool_mem_top_k, ) def get_relevant_subgraph( diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/recall.py b/src/memos/memories/textual/tree_text_memory/retrieve/recall.py index 5dfbde704..dea83887e 100644 --- a/src/memos/memories/textual/tree_text_memory/retrieve/recall.py +++ b/src/memos/memories/textual/tree_text_memory/retrieve/recall.py @@ -59,7 +59,13 @@ def retrieve( Returns: list: Combined memory items. """ - if memory_scope not in ["WorkingMemory", "LongTermMemory", "UserMemory"]: + if memory_scope not in [ + "WorkingMemory", + "LongTermMemory", + "UserMemory", + "ToolSchemaMemory", + "ToolTrajectoryMemory", + ]: raise ValueError(f"Unsupported memory scope: {memory_scope}") if memory_scope == "WorkingMemory": diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py index 830b915c1..0666f1d86 100644 --- a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py +++ b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py @@ -76,6 +76,8 @@ def retrieve( search_filter: dict | None = None, search_priority: dict | None = None, user_name: str | None = None, + search_tool_memory: bool = False, + tool_mem_top_k: int = 6, **kwargs, ) -> list[tuple[TextualMemoryItem, float]]: logger.info( @@ -100,6 +102,8 @@ def retrieve( search_filter, search_priority, user_name, + search_tool_memory, + tool_mem_top_k, ) return results @@ -109,10 +113,14 @@ def post_retrieve( top_k: int, user_name: str | None = None, info=None, + search_tool_memory: bool = False, + tool_mem_top_k: int = 6, plugin=False, ): deduped = self._deduplicate_results(retrieved_results) - final_results = self._sort_and_trim(deduped, top_k, plugin) + final_results = self._sort_and_trim( + deduped, top_k, plugin, search_tool_memory, tool_mem_top_k + ) self._update_usage_history(final_results, info, user_name) return final_results @@ -127,6 +135,8 @@ def search( search_filter: dict | None = None, search_priority: dict | None = None, user_name: str | None = None, + search_tool_memory: bool = False, + tool_mem_top_k: int = 6, **kwargs, ) -> list[TextualMemoryItem]: """ @@ -171,6 +181,8 @@ def search( search_filter=search_filter, search_priority=search_priority, user_name=user_name, + search_tool_memory=search_tool_memory, + tool_mem_top_k=tool_mem_top_k, ) final_results = self.post_retrieve( @@ -179,6 +191,8 @@ def search( user_name=user_name, info=None, plugin=kwargs.get("plugin", False), + search_tool_memory=search_tool_memory, + tool_mem_top_k=tool_mem_top_k, ) logger.info(f"[SEARCH] Done. Total {len(final_results)} results.") @@ -272,6 +286,8 @@ def _retrieve_paths( search_filter: dict | None = None, search_priority: dict | None = None, user_name: str | None = None, + search_tool_memory: bool = False, + tool_mem_top_k: int = 6, ): """Run A/B/C retrieval paths in parallel""" tasks = [] @@ -324,6 +340,22 @@ def _retrieve_paths( user_name, ) ) + if search_tool_memory: + tasks.append( + executor.submit( + self._retrieve_from_tool_memory, + query, + parsed_goal, + query_embedding, + top_k, + memory_type, + search_filter, + search_priority, + user_name, + id_filter, + mode=mode, + ) + ) results = [] for t in tasks: @@ -498,6 +530,98 @@ def _retrieve_from_internet( parsed_goal=parsed_goal, ) + # --- Path D + @timed + def _retrieve_from_tool_memory( + self, + query, + parsed_goal, + query_embedding, + top_k, + memory_type, + search_filter: dict | None = None, + search_priority: dict | None = None, + user_name: str | None = None, + id_filter: dict | None = None, + mode: str = "fast", + ): + """Retrieve and rerank from ToolMemory""" + results = { + "ToolSchemaMemory": [], + "ToolTrajectoryMemory": [], + } + tasks = [] + + # chain of thinking + cot_embeddings = [] + if self.vec_cot: + queries = self._cot_query(query, mode=mode, context=parsed_goal.context) + if len(queries) > 1: + cot_embeddings = self.embedder.embed(queries) + cot_embeddings.extend(query_embedding) + else: + cot_embeddings = query_embedding + + with ContextThreadPoolExecutor(max_workers=2) as executor: + if memory_type in ["All", "ToolSchemaMemory"]: + tasks.append( + executor.submit( + self.graph_retriever.retrieve, + query=query, + parsed_goal=parsed_goal, + query_embedding=cot_embeddings, + top_k=top_k * 2, + memory_scope="ToolSchemaMemory", + search_filter=search_filter, + search_priority=search_priority, + user_name=user_name, + id_filter=id_filter, + use_fast_graph=self.use_fast_graph, + ) + ) + if memory_type in ["All", "ToolTrajectoryMemory"]: + tasks.append( + executor.submit( + self.graph_retriever.retrieve, + query=query, + parsed_goal=parsed_goal, + query_embedding=cot_embeddings, + top_k=top_k * 2, + memory_scope="ToolTrajectoryMemory", + search_filter=search_filter, + search_priority=search_priority, + user_name=user_name, + id_filter=id_filter, + use_fast_graph=self.use_fast_graph, + ) + ) + + # Collect results from all tasks + for task in tasks: + rsp = task.result() + if rsp and rsp[0].metadata.memory_type == "ToolSchemaMemory": + results["ToolSchemaMemory"].extend(rsp) + elif rsp and rsp[0].metadata.memory_type == "ToolTrajectoryMemory": + results["ToolTrajectoryMemory"].extend(rsp) + + schema_reranked = self.reranker.rerank( + query=query, + query_embedding=query_embedding[0], + graph_results=results["ToolSchemaMemory"], + top_k=top_k, + parsed_goal=parsed_goal, + search_filter=search_filter, + ) + trajectory_reranked = self.reranker.rerank( + query=query, + query_embedding=query_embedding[0], + graph_results=results["ToolTrajectoryMemory"], + top_k=top_k, + parsed_goal=parsed_goal, + search_filter=search_filter, + ) + return schema_reranked + trajectory_reranked + @timed def _retrieve_simple( self, @@ -554,11 +678,41 @@ def _deduplicate_results(self, results): return list(deduped.values()) @timed - def _sort_and_trim(self, results, top_k, plugin=False): + def _sort_and_trim( + self, results, top_k, plugin=False, search_tool_memory=False, tool_mem_top_k=6 + ): """Sort results by score and trim to top_k""" + final_items = [] + if search_tool_memory: + tool_results = [ + (item, score) + for item, score in results + if item.metadata.memory_type in ["ToolSchemaMemory", "ToolTrajectoryMemory"] + ] + sorted_tool_results = sorted(tool_results, key=lambda pair: pair[1], reverse=True)[ + :tool_mem_top_k + ] + for item, score in sorted_tool_results: + if plugin and round(score, 2) == 0.00: + continue + meta_data = item.metadata.model_dump() + meta_data["relativity"] = score + final_items.append( + TextualMemoryItem( + id=item.id, + memory=item.memory, + metadata=SearchedTreeNodeTextualMemoryMetadata(**meta_data), + ) + ) + # separate textual results + results = [ + (item, score) + for item, score in results + if item.metadata.memory_type not in ["ToolSchemaMemory", "ToolTrajectoryMemory"] + ] sorted_results = sorted(results, key=lambda pair: pair[1], reverse=True)[:top_k] - final_items = [] + for item, score in sorted_results: if plugin and round(score, 2) == 0.00: continue diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index c6fe588cb..1ddd2b1b7 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -11,6 +11,7 @@ from memos.api.handlers.formatters_handler import ( format_memory_item, post_process_pref_mem, + post_process_textual_mem, ) from memos.context.context import ContextThreadPoolExecutor from memos.log import get_logger @@ -109,6 +110,7 @@ def search_memories(self, search_req: APISearchRequest) -> dict[str, Any]: "para_mem": [], "pref_mem": [], "pref_note": "", + "tool_mem": [], } # Determine search mode @@ -123,11 +125,10 @@ def search_memories(self, search_req: APISearchRequest) -> dict[str, Any]: pref_formatted_memories = pref_future.result() # Build result - memories_result["text_mem"].append( - { - "cube_id": self.cube_id, - "memories": text_formatted_memories, - } + memories_result = post_process_textual_mem( + memories_result, + text_formatted_memories, + self.cube_id, ) memories_result = post_process_pref_mem( @@ -278,6 +279,8 @@ def _fine_search( Returns: List of enhanced search results """ + # TODO: support tool memory search in future + logger.info(f"Fine strategy: {FINE_STRATEGY}") if FINE_STRATEGY == FineStrategy.DEEP_SEARCH: return self._deep_search(search_req=search_req, user_context=user_context) @@ -375,6 +378,9 @@ def _search_pref( """ if os.getenv("ENABLE_PREFERENCE_MEMORY", "false").lower() != "true": return [] + if not search_req.include_preference: + return [] + logger.info(f"search_req.filter for preference memory: {search_req.filter}") logger.info(f"type of pref_mem: {type(self.naive_mem_cube.pref_mem)}") try: @@ -427,6 +433,8 @@ def _fast_search( "chat_history": search_req.chat_history, }, plugin=plugin, + search_tool_memory=search_req.search_tool_memory, + tool_mem_top_k=search_req.tool_mem_top_k, ) formatted_memories = [format_memory_item(data) for data in search_results] From da9d843382321c223ba891030477391289237989 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 3 Dec 2025 17:30:17 +0800 Subject: [PATCH 12/13] tool search --- src/memos/api/product_models.py | 22 +++++++++---------- .../read_multi_modal/system_parser.py | 9 ++++++++ .../tree_text_memory/organize/manager.py | 9 ++++---- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/memos/api/product_models.py b/src/memos/api/product_models.py index acdb773e9..4ad4016bc 100644 --- a/src/memos/api/product_models.py +++ b/src/memos/api/product_models.py @@ -6,7 +6,7 @@ # Import message types from core types module from memos.log import get_logger -from memos.types import MessageDict, PermissionDict, SearchMode +from memos.types import MessageList, MessagesType, PermissionDict, SearchMode logger = get_logger(__name__) @@ -56,7 +56,7 @@ class Message(BaseModel): class MemoryCreate(BaseRequest): user_id: str = Field(..., description="User ID") - messages: list | None = Field(None, description="List of messages to store.") + messages: MessageList | None = Field(None, description="List of messages to store.") memory_content: str | None = Field(None, description="Content to store as memory") doc_path: str | None = Field(None, description="Path to document to store") mem_cube_id: str | None = Field(None, description="ID of the memory cube") @@ -83,7 +83,7 @@ class ChatRequest(BaseRequest): writable_cube_ids: list[str] | None = Field( None, description="List of cube IDs user can write for multi-cube chat" ) - history: list | None = Field(None, description="Chat history") + history: MessageList | None = Field(None, description="Chat history") mode: SearchMode = Field(SearchMode.FAST, description="search mode: fast, fine, or mixture") system_prompt: str | None = Field(None, description="Base system prompt to use for chat") top_k: int = Field(10, description="Number of results to return") @@ -165,7 +165,7 @@ class ChatCompleteRequest(BaseRequest): user_id: str = Field(..., description="User ID") query: str = Field(..., description="Chat query message") mem_cube_id: str | None = Field(None, description="Cube ID to use for chat") - history: list | None = Field(None, description="Chat history") + history: MessageList | None = Field(None, description="Chat history") internet_search: bool = Field(False, description="Whether to use internet search") system_prompt: str | None = Field(None, description="Base prompt to use for chat") top_k: int = Field(10, description="Number of results to return") @@ -251,7 +251,7 @@ class MemoryCreateRequest(BaseRequest): """Request model for creating memories.""" user_id: str = Field(..., description="User ID") - messages: str | list | None = Field(None, description="List of messages to store.") + messages: str | MessagesType | None = Field(None, description="List of messages to store.") memory_content: str | None = Field(None, description="Memory content to store") doc_path: str | None = Field(None, description="Path to document to store") mem_cube_id: str | None = Field(None, description="Cube ID") @@ -375,7 +375,7 @@ class APISearchRequest(BaseRequest): ) # ==== Context ==== - chat_history: list | None = Field( + chat_history: MessageList | None = Field( None, description=( "Historical chat messages used internally by algorithms. " @@ -505,7 +505,7 @@ class APIADDRequest(BaseRequest): ) # ==== Input content ==== - messages: str | list | None = Field( + messages: MessagesType | None = Field( None, description=( "List of messages to store. Supports: " @@ -521,7 +521,7 @@ class APIADDRequest(BaseRequest): ) # ==== Chat history ==== - chat_history: list | None = Field( + chat_history: MessageList | None = Field( None, description=( "Historical chat messages used internally by algorithms. " @@ -651,7 +651,7 @@ class APIFeedbackRequest(BaseRequest): "default_session", description="Session ID for soft-filtering memories" ) task_id: str | None = Field(None, description="Task ID for monitering async tasks") - history: list[MessageDict] | None = Field(..., description="Chat history") + history: MessageList | None = Field(..., description="Chat history") retrieved_memory_ids: list[str] | None = Field( None, description="Retrieved memory ids at last turn" ) @@ -686,7 +686,7 @@ class APIChatCompleteRequest(BaseRequest): writable_cube_ids: list[str] | None = Field( None, description="List of cube IDs user can write for multi-cube chat" ) - history: list | None = Field(None, description="Chat history") + history: MessageList | None = Field(None, description="Chat history") mode: SearchMode = Field(SearchMode.FAST, description="search mode: fast, fine, or mixture") system_prompt: str | None = Field(None, description="Base system prompt to use for chat") top_k: int = Field(10, description="Number of results to return") @@ -755,7 +755,7 @@ class SuggestionRequest(BaseRequest): user_id: str = Field(..., description="User ID") mem_cube_id: str = Field(..., description="Cube ID") language: Literal["zh", "en"] = Field("zh", description="Language for suggestions") - message: list | None = Field(None, description="List of messages to store.") + message: MessagesType | None = Field(None, description="List of messages to store.") # ─── MemOS Client Response Models ────────────────────────────────────────────── diff --git a/src/memos/mem_reader/read_multi_modal/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py index e08279dc1..c3961c73a 100644 --- a/src/memos/mem_reader/read_multi_modal/system_parser.py +++ b/src/memos/mem_reader/read_multi_modal/system_parser.py @@ -126,12 +126,21 @@ def parse_fine( logger.warning(f"[SystemParser] Tool schema must be a list[dict]: {content}") return [] + info_ = info.copy() + user_id = info_.pop("user_id", "") + session_id = info_.pop("session_id", "") + return [ TextualMemoryItem( id=str(uuid.uuid4()), memory=json.dumps(schema), metadata=TreeNodeTextualMemoryMetadata( + user_id=user_id, + session_id=session_id, memory_type="ToolSchemaMemory", + status="activated", + embedding=self.embedder.embed([json.dumps(schema)])[0], + info=info_, ), ) for schema in tool_schema diff --git a/src/memos/memories/textual/tree_text_memory/organize/manager.py b/src/memos/memories/textual/tree_text_memory/organize/manager.py index 31cf89ae1..3226f7ca0 100644 --- a/src/memos/memories/textual/tree_text_memory/organize/manager.py +++ b/src/memos/memories/textual/tree_text_memory/organize/manager.py @@ -181,10 +181,11 @@ def _process_memory(self, memory: TextualMemoryItem, user_name: str | None = Non working_id = str(uuid.uuid4()) with ContextThreadPoolExecutor(max_workers=2, thread_name_prefix="mem") as ex: - f_working = ex.submit( - self._add_memory_to_db, memory, "WorkingMemory", user_name, working_id - ) - futures.append(("working", f_working)) + if memory.metadata.memory_type not in ("ToolSchemaMemory", "ToolTrajectoryMemory"): + f_working = ex.submit( + self._add_memory_to_db, memory, "WorkingMemory", user_name, working_id + ) + futures.append(("working", f_working)) if memory.metadata.memory_type in ( "LongTermMemory", From 96a9dfdbc59dce8c4540c7fee785b0f026ba35d0 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 3 Dec 2025 19:37:32 +0800 Subject: [PATCH 13/13] add split chunck in system and tool --- docker/requirements.txt | 1 + .../read_multi_modal/system_parser.py | 25 ++++++++++++++++--- .../read_multi_modal/tool_parser.py | 23 ++++++++++++++--- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index d3268edae..21f246599 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -160,3 +160,4 @@ xlrd==2.0.2 xlsxwriter==3.2.5 prometheus-client==0.23.1 pymilvus==2.5.12 +langchain-text-splitters==1.0.0 diff --git a/src/memos/mem_reader/read_multi_modal/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py index c3961c73a..3f467d649 100644 --- a/src/memos/mem_reader/read_multi_modal/system_parser.py +++ b/src/memos/mem_reader/read_multi_modal/system_parser.py @@ -95,17 +95,34 @@ def parse_fast( ) source = self.create_source(message, info) - return [ - TextualMemoryItem( - memory=content_wo_tool_schema, + + # Extract info fields + info_ = info.copy() + user_id = info_.pop("user_id", "") + session_id = info_.pop("session_id", "") + + # Split parsed text into chunks + content_chunks = self._split_text(content_wo_tool_schema) + + memory_items = [] + for _chunk_idx, chunk_text in enumerate(content_chunks): + if not chunk_text.strip(): + continue + + memory_item = TextualMemoryItem( + memory=chunk_text, metadata=TreeNodeTextualMemoryMetadata( + user_id=user_id, + session_id=session_id, memory_type="LongTermMemory", # only choce long term memory for system messages as a placeholder status="activated", tags=["mode:fast"], sources=[source], + info=info_, ), ) - ] + memory_items.append(memory_item) + return memory_items def parse_fine( self, diff --git a/src/memos/mem_reader/read_multi_modal/tool_parser.py b/src/memos/mem_reader/read_multi_modal/tool_parser.py index cdd3620cb..09bd9e9d0 100644 --- a/src/memos/mem_reader/read_multi_modal/tool_parser.py +++ b/src/memos/mem_reader/read_multi_modal/tool_parser.py @@ -158,17 +158,32 @@ def parse_fast( return [] sources = self.create_source(message, info) - return [ - TextualMemoryItem( - memory=line, + + # Extract info fields + info_ = info.copy() + user_id = info_.pop("user_id", "") + session_id = info_.pop("session_id", "") + + content_chunks = self._split_text(line) + memory_items = [] + for _chunk_idx, chunk_text in enumerate(content_chunks): + if not chunk_text.strip(): + continue + + memory_item = TextualMemoryItem( + memory=chunk_text, metadata=TreeNodeTextualMemoryMetadata( + user_id=user_id, + session_id=session_id, memory_type="LongTermMemory", # only choce long term memory for tool messages as a placeholder status="activated", tags=["mode:fast"], sources=sources, + info=info_, ), ) - ] + memory_items.append(memory_item) + return memory_items def parse_fine( self,