diff --git a/src/memos/api/routers/server_router.py b/src/memos/api/routers/server_router.py
index c60e84253..a4052d313 100644
--- a/src/memos/api/routers/server_router.py
+++ b/src/memos/api/routers/server_router.py
@@ -90,6 +90,7 @@
status_tracker = TaskStatusTracker(redis_client=redis_client)
embedder = components["embedder"]
graph_db = components["graph_db"]
+vector_db = components["vector_db"]
# =============================================================================
@@ -359,6 +360,13 @@ def get_user_names_by_memory_ids(request: GetUserNamesByMemoryIdsRequest):
),
)
result = graph_db.get_user_names_by_memory_ids(memory_ids=request.memory_ids)
+ if vector_db:
+ prefs = []
+ for collection_name in ["explicit_preference", "implicit_preference"]:
+ prefs.extend(
+ vector_db.get_by_ids(collection_name=collection_name, ids=request.memory_ids)
+ )
+ result.update({pref.id: pref.payload.get("mem_cube_id", None) for pref in prefs})
return GetUserNamesByMemoryIdsResponse(
code=200,
message="Successfully",
diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py
index 2ed1af53e..3bf6d4927 100644
--- a/src/memos/mem_reader/multi_modal_struct.py
+++ b/src/memos/mem_reader/multi_modal_struct.py
@@ -1,5 +1,6 @@
import concurrent.futures
import json
+import re
import traceback
from typing import Any
@@ -547,7 +548,11 @@ def _process_tool_trajectory_fine(
for fast_item in fast_memory_items:
# Extract memory text (string content)
mem_str = fast_item.memory or ""
- if not mem_str.strip() or "tool:" not in mem_str:
+ if not mem_str.strip() or (
+ "tool:" not in mem_str
+ and "[tool_calls]:" not in mem_str
+ and not re.search(r".*?", mem_str, re.DOTALL)
+ ):
continue
try:
resp = self._get_llm_tool_trajectory_response(mem_str)
@@ -563,6 +568,8 @@ def _process_tool_trajectory_fine(
value=m.get("trajectory", ""),
info=info,
memory_type=memory_type,
+ correctness=m.get("correctness", ""),
+ experience=m.get("experience", ""),
tool_used_status=m.get("tool_used_status", []),
)
fine_memory_items.append(node)
@@ -606,16 +613,22 @@ def _process_multi_modal_data(
if mode == "fast":
return fast_memory_items
else:
- # Part A: call llm
+ # Part A: call llm in parallel using thread pool
fine_memory_items = []
- fine_memory_items_string_parser = self._process_string_fine(
- fast_memory_items, info, custom_tags
- )
- fine_memory_items.extend(fine_memory_items_string_parser)
- fine_memory_items_tool_trajectory_parser = self._process_tool_trajectory_fine(
- fast_memory_items, info
- )
+ with ContextThreadPoolExecutor(max_workers=2) as executor:
+ future_string = executor.submit(
+ self._process_string_fine, fast_memory_items, info, custom_tags
+ )
+ future_tool = executor.submit(
+ self._process_tool_trajectory_fine, fast_memory_items, info
+ )
+
+ # Collect results
+ fine_memory_items_string_parser = future_string.result()
+ fine_memory_items_tool_trajectory_parser = future_tool.result()
+
+ fine_memory_items.extend(fine_memory_items_string_parser)
fine_memory_items.extend(fine_memory_items_tool_trajectory_parser)
# Part B: get fine multimodal items
@@ -658,13 +671,18 @@ def _process_transfer_multi_modal_data(
}
fine_memory_items = []
- # Part A: call llm
- fine_memory_items_string_parser = self._process_string_fine([raw_node], info, custom_tags)
- fine_memory_items.extend(fine_memory_items_string_parser)
+ # Part A: call llm in parallel using thread pool
+ with ContextThreadPoolExecutor(max_workers=2) as executor:
+ future_string = executor.submit(
+ self._process_string_fine, [raw_node], info, custom_tags
+ )
+ future_tool = executor.submit(self._process_tool_trajectory_fine, [raw_node], info)
- fine_memory_items_tool_trajectory_parser = self._process_tool_trajectory_fine(
- [raw_node], info
- )
+ # Collect results
+ fine_memory_items_string_parser = future_string.result()
+ fine_memory_items_tool_trajectory_parser = future_tool.result()
+
+ fine_memory_items.extend(fine_memory_items_string_parser)
fine_memory_items.extend(fine_memory_items_tool_trajectory_parser)
# Part B: get fine multimodal items
diff --git a/src/memos/mem_reader/read_multi_modal/system_parser.py b/src/memos/mem_reader/read_multi_modal/system_parser.py
index deb2a9832..dfffb4626 100644
--- a/src/memos/mem_reader/read_multi_modal/system_parser.py
+++ b/src/memos/mem_reader/read_multi_modal/system_parser.py
@@ -42,9 +42,10 @@ def create_source(
info: dict[str, Any],
) -> SourceMessage:
"""Create SourceMessage from system message."""
- content = message["content"]
+
+ content = message.get("content", "")
if isinstance(content, dict):
- content = content["text"]
+ content = content.get("text", "")
content_wo_tool_schema = re.sub(
r"(.*?)",
@@ -84,17 +85,146 @@ def parse_fast(
info: dict[str, Any],
**kwargs,
) -> list[TextualMemoryItem]:
- content = message["content"]
+ content = message.get("content", "")
if isinstance(content, dict):
- content = content["text"]
+ content = content.get("text", "")
- # Replace tool_schema content with "omitted" in remaining content
- content_wo_tool_schema = re.sub(
- r"(.*?)",
- r"omitted",
- content,
- flags=re.DOTALL,
- )
+ # Find first tool_schema block
+ tool_schema_pattern = r"(.*?)"
+ match = re.search(tool_schema_pattern, content, flags=re.DOTALL)
+
+ if match:
+ original_text = match.group(0) # Complete ... block
+ schema_content = match.group(1) # Content between the tags
+
+ # Parse tool schema
+ try:
+ tool_schema = json.loads(schema_content)
+ assert isinstance(tool_schema, list), "Tool schema must be a list[dict]"
+ except json.JSONDecodeError:
+ try:
+ tool_schema = ast.literal_eval(schema_content)
+ assert isinstance(tool_schema, list), "Tool schema must be a list[dict]"
+ except (ValueError, SyntaxError, AssertionError):
+ logger.warning(
+ f"[SystemParser] Failed to parse tool schema with both JSON and ast.literal_eval: {schema_content[:100]}..."
+ )
+ tool_schema = None
+ except AssertionError:
+ logger.warning(
+ f"[SystemParser] Tool schema must be a list[dict]: {schema_content[:100]}..."
+ )
+ tool_schema = None
+
+ # Process and replace
+ if tool_schema is not None:
+
+ def remove_descriptions(obj):
+ """Recursively remove all 'description' keys from a nested dict/list structure."""
+ if isinstance(obj, dict):
+ return {
+ k: remove_descriptions(v) for k, v in obj.items() if k != "description"
+ }
+ elif isinstance(obj, list):
+ return [remove_descriptions(item) for item in obj]
+ else:
+ return obj
+
+ def keep_first_layer_params(obj):
+ """Only keep first layer parameter information, remove nested parameters."""
+ if isinstance(obj, list):
+ return [keep_first_layer_params(item) for item in obj]
+ elif isinstance(obj, dict):
+ result = {}
+ for k, v in obj.items():
+ if k == "properties" and isinstance(v, dict):
+ # For properties, only keep first layer parameter names and types
+ first_layer_props = {}
+ for param_name, param_info in v.items():
+ if isinstance(param_info, dict):
+ # Only keep type and basic info, remove nested properties
+ first_layer_props[param_name] = {
+ key: val
+ for key, val in param_info.items()
+ if key in ["type", "enum", "required"]
+ and key != "properties"
+ }
+ else:
+ first_layer_props[param_name] = param_info
+ result[k] = first_layer_props
+ elif k == "parameters" and isinstance(v, dict):
+ # Process parameters object but only keep first layer
+ result[k] = keep_first_layer_params(v)
+ elif isinstance(v, dict | list) and k != "properties":
+ result[k] = keep_first_layer_params(v)
+ else:
+ result[k] = v
+ return result
+ else:
+ return obj
+
+ def format_tool_schema_readable(tool_schema):
+ """Convert tool schema to readable format: tool_name: [param1 (type1), ...](required: ...)"""
+ lines = []
+ for tool in tool_schema:
+ if not tool:
+ continue
+
+ # Handle both new format and old-style OpenAI function format
+ if tool.get("type") == "function" and "function" in tool:
+ tool_info = tool.get("function")
+ if not tool_info:
+ continue
+ else:
+ tool_info = tool
+
+ tool_name = tool_info.get("name", "unknown")
+ params_obj = tool_info.get("parameters", {})
+ properties = params_obj.get("properties", {})
+ required = params_obj.get("required", [])
+
+ # Format parameters
+ param_strs = []
+ for param_name, param_info in properties.items():
+ if isinstance(param_info, dict):
+ param_type = param_info.get("type", "any")
+ # Handle enum
+ if "enum" in param_info and param_info["enum"] is not None:
+ # Ensure all enum values are strings
+ enum_values = [str(v) for v in param_info["enum"]]
+ param_type = f"{param_type}[{', '.join(enum_values)}]"
+ param_strs.append(f"{param_name} ({param_type})")
+ else:
+ param_strs.append(f"{param_name} (any)")
+
+ # Format required parameters
+ # Ensure all required parameter names are strings
+ required_strs = [str(r) for r in required] if required else []
+ required_str = (
+ f"(required: {', '.join(required_strs)})" if required_strs else ""
+ )
+
+ # Construct the line
+ params_part = f"[{', '.join(param_strs)}]" if param_strs else "[]"
+ line = f"{tool_name}: {params_part}{required_str}"
+ lines.append(line)
+
+ return "\n".join(lines)
+
+ # First keep only first layer params, then remove descriptions
+ simple_tool_schema = keep_first_layer_params(tool_schema)
+ simple_tool_schema = remove_descriptions(simple_tool_schema)
+ # change to readable format
+ readable_schema = format_tool_schema_readable(simple_tool_schema)
+
+ processed_text = f"{readable_schema}"
+ content = content.replace(original_text, processed_text, 1)
+
+ parts = ["system: "]
+ if message.get("chat_time"):
+ parts.append(f"[{message.get('chat_time')}]: ")
+ prefix = "".join(parts)
+ msg_line = f"{prefix}{content}\n"
source = self.create_source(message, info)
@@ -104,7 +234,7 @@ def parse_fast(
session_id = info_.pop("session_id", "")
# Split parsed text into chunks
- content_chunks = self._split_text(content_wo_tool_schema)
+ content_chunks = self._split_text(msg_line)
memory_items = []
for _chunk_idx, chunk_text in enumerate(content_chunks):
@@ -132,9 +262,9 @@ def parse_fine(
info: dict[str, Any],
**kwargs,
) -> list[TextualMemoryItem]:
- content = message["content"]
+ content = message.get("content", "")
if isinstance(content, dict):
- content = content["text"]
+ content = content.get("text", "")
try:
tool_schema = json.loads(content)
assert isinstance(tool_schema, list), "Tool schema must be a list[dict]"
diff --git a/src/memos/memories/textual/preference.py b/src/memos/memories/textual/preference.py
index a34315918..78f4d6e28 100644
--- a/src/memos/memories/textual/preference.py
+++ b/src/memos/memories/textual/preference.py
@@ -248,7 +248,7 @@ def get_all(self) -> list[TextualMemoryItem]:
Returns:
list[TextualMemoryItem]: List of all memories.
"""
- all_collections = self.vector_db.list_collections()
+ all_collections = ["explicit_preference", "implicit_preference"]
all_memories = {}
for collection_name in all_collections:
items = self.vector_db.get_all(collection_name)
diff --git a/src/memos/memories/textual/simple_preference.py b/src/memos/memories/textual/simple_preference.py
index ee37d638c..cc1781f06 100644
--- a/src/memos/memories/textual/simple_preference.py
+++ b/src/memos/memories/textual/simple_preference.py
@@ -90,7 +90,7 @@ def get_with_collection_name(
return None
return TextualMemoryItem(
id=res.id,
- memory=res.payload.get("dialog_str", ""),
+ memory=res.memory,
metadata=PreferenceTextualMemoryMetadata(**res.payload),
)
except Exception as e:
@@ -116,7 +116,7 @@ def get_by_ids_with_collection_name(
return [
TextualMemoryItem(
id=memo.id,
- memory=memo.payload.get("dialog_str", ""),
+ memory=memo.memory,
metadata=PreferenceTextualMemoryMetadata(**memo.payload),
)
for memo in res
@@ -132,14 +132,14 @@ def get_all(self) -> list[TextualMemoryItem]:
Returns:
list[TextualMemoryItem]: List of all memories.
"""
- all_collections = self.vector_db.list_collections()
+ all_collections = ["explicit_preference", "implicit_preference"]
all_memories = {}
for collection_name in all_collections:
items = self.vector_db.get_all(collection_name)
all_memories[collection_name] = [
TextualMemoryItem(
id=memo.id,
- memory=memo.payload.get("dialog_str", ""),
+ memory=memo.memory,
metadata=PreferenceTextualMemoryMetadata(**memo.payload),
)
for memo in items
diff --git a/src/memos/templates/tool_mem_prompts.py b/src/memos/templates/tool_mem_prompts.py
index 7d5363956..f19a7c32a 100644
--- a/src/memos/templates/tool_mem_prompts.py
+++ b/src/memos/templates/tool_mem_prompts.py
@@ -1,26 +1,46 @@
TOOL_TRAJECTORY_PROMPT_ZH = """
-你是一个专业的工具调用轨迹提取专家。你的任务是从给定的对话消息中提取完整的工具调用轨迹经验。
+你是一个专业的工具经验提取专家。你的任务是从给定的对话消息中提取完整的工具调用轨迹经验。
-## 提取规则:
-1. 只有当对话中存在有价值的工具调用过程时才进行提取
-2. 有价值的轨迹至少包含以下元素:
- - 用户的问题(user message)
- - 助手的工具调用尝试(assistant message with tool_calls)
- - 工具的执行结果(tool message with tool_call_id and content,无论成功或失败)
- - 助手的响应(assistant message,无论是否给出最终答案)
+## 分析判断步骤:
+**步骤1:判断任务完成度**
+根据用户反馈,判定correctness:success(成功)或 failed(失败),用户反馈决定权大于执行结果,用户反馈有误,则判定为failed
+
+**步骤2:成功轨迹(success)- 经验提炼**
+从成功模式中提炼通用原则或规则,采用"when...then..."结构:
+- when: 明确描述触发该经验的场景特征(任务类型、工具环境、参数特征等)
+- then: 总结有效的参数模式、调用策略、最佳实践
+注意:经验是解决整个轨迹问题级别的,不仅仅针对单个工具
+
+**步骤3:失败轨迹(failed)- 错误分析与经验提炼**
+3.1 工具需求判断
+ - 任务是否需要工具?(需要/直接回答/误调用)
+3.2 工具调用检查
+ - 工具存在性:是否在system中提供
+ - 工具选择:是否选对工具
+ - 参数正确性:是否符合类型定义
+ - 幻觉检测:是否调用不存在的工具
+3.3 错误根因定位
+ 结合消息中的错误反馈信息和上述分析,精准输出根本原因
+3.4 经验提炼(核心)
+ 从失败模式中提炼通用原则或规则,采用"when...then..."结构:
+ - when: 明确描述触发该经验的场景特征(任务类型、工具环境、参数特征等)
+ - then: 给出避免错误的通用策略、正确调用方式或决策规则
+ 注意:经验是解决整个轨迹问题级别的,不仅仅针对单个工具
## 输出格式:
返回一个JSON数组,格式如下:
+
```json
[
{
- "trajectory": "自然语言输出包含'任务、使用的工具、工具观察、最终回答'的完整精炼的总结,体现顺序",
+ "correctness": "success 或 failed",
+ "trajectory": "精炼完整的自然语言总结,包含:[任务(用户任务) -> 执行动作(调用的工具/直接回答) -> 执行结果] (可能多轮) -> 最终回答",
+ "experience": "采用when...then...格式,例如:'when 遇到XX的任务时,应该YY'",
"tool_used_status": [
{
- "used_tool": "工具名1",
+ "used_tool": "工具名称(如果调用了工具)",
"success_rate": "0.0-1.0之间的数值,表示该工具在本次轨迹中的成功率",
- "error_type": "调用失败时的错误类型和描述,成功时为空字符串",
- "experience": "该工具的使用经验,比如常见的参数模式、执行特点、结果解读方式等"
+ "error_type": "调用失败时的错误类型和描述,成功时为空字符串"
}
]
}
@@ -28,42 +48,71 @@
```
## 注意事项:
-- 如果对话中没有完整的工具调用轨迹,返回空数组
- 每个轨迹必须是独立的完整过程
- 一个轨迹中可能涉及多个工具的使用,每个工具在tool_used_status中独立记录
+- 如果没有调用工具,tool_used_status为空数组[]
+- 如果多条轨迹存在顺序依赖关系,需要将它们视为一条轨迹
- 只提取事实内容,不要添加任何解释或额外信息
- 确保返回的是有效的JSON格式
+- 输出的trajectory需要按照messages的发展顺序排列
+- experience必须是通用的、可复用的经验规则,而不是针对具体案例的描述
+- 无论成功或失败,都要提炼经验并使用when...then...格式
-请分析以下对话消息并提取工具调用轨迹:
-
+请分析以下对话消息并提取工具调用轨迹,基于以下对话消息:
+
{messages}
-
+
"""
TOOL_TRAJECTORY_PROMPT_EN = """
-You are a professional tool call trajectory extraction expert. Your task is to extract valuable tool call trajectory experiences from given conversation messages.
+You are a professional tool experience extraction expert. Your task is to extract complete tool call trajectory experiences from given conversation messages.
+
+## Analysis and Judgment Steps:
+
+**Step 1: Assess Task Completion**
+Determine correctness based on user feedback: success or failed, user feedback has higher priority than execution results, if user feedback is incorrect, then determine as failed
+
+**Step 2: Successful Trajectory (success) - Experience Extraction**
+Extract general principles or rules from success patterns, using "when...then..." structure:
+- when: clearly describe the scenario characteristics that trigger this experience (task type, tool environment, parameter characteristics, etc.)
+- then: summarize effective parameter patterns, calling strategies, and best practices
+Note: Experience is at the trajectory-level problem-solving, not just for a single tool
+
+**Step 3: Failed Trajectory (failed) - Error Analysis and Experience Extraction**
-## Extraction Rules:
-1. Only extract when there are valuable tool calling processes in the conversation
-2. Valuable trajectories must contain at least the following elements:
- - User's question (user message)
- - Assistant's tool call attempt (assistant message with tool_calls)
- - Tool execution results (tool message with tool_call_id and content, regardless of success or failure)
- - Assistant's response (assistant message, whether or not a final answer is given)
+3.1 Tool Requirement Assessment
+ - Does the task require tools? (required/direct answer/unnecessary call)
+
+3.2 Tool Call Verification
+ - Tool availability: provided in system?
+ - Tool selection: correct tool chosen?
+ - Parameter correctness: conform to type definitions?
+ - Hallucination detection: calling non-existent tools?
+
+3.3 Root Cause Identification
+ Combine error feedback from messages with above analysis to precisely output root cause
+
+3.4 Experience Extraction (Core)
+ Extract general principles or rules from failure patterns, using "when...then..." structure:
+ - when: clearly describe the scenario characteristics that trigger this experience (task type, tool environment, parameter characteristics, etc.)
+ - then: provide general strategies to avoid errors, correct calling approaches, or decision rules
+ Note: Experience is at the trajectory-level problem-solving, not just for a single tool
## Output Format:
Return a JSON array in the following format:
+
```json
[
{
- "trajectory": "Natural language summary containing 'task, tools used, tool observations, final answer' in a complete and refined manner, reflecting the sequence",
+ "correctness": "success or failed",
+ "trajectory": "Concise and complete natural language summary including: [task (user task) -> execution action (tool called/direct answer) -> execution result] (possibly multiple rounds) -> final answer",
+ "experience": "Use when...then... format, e.g., 'when encountering XX tasks, should do YY'",
"tool_used_status": [
{
- "used_tool": "Tool Name 1",
- "success_rate": "Numerical value between 0.0-1.0, indicating the success rate of this tool in the current trajectory",
- "error_type": "Error type and description when call fails, empty string when successful",
- "experience": "Usage experience of this tool, such as common parameter patterns, execution characteristics, result interpretation methods, etc."
+ "used_tool": "Tool name (if tool was called)",
+ "success_rate": "Numerical value between 0.0-1.0, indicating the success rate of this tool in current trajectory",
+ "error_type": "Error type and description when call fails, empty string when successful"
}
]
}
@@ -71,14 +120,151 @@
```
## Notes:
-- If there are no complete tool call trajectories in the conversation, return an empty array
- Each trajectory must be an independent complete process
-- Multiple tools may be used in one trajectory, each tool is recorded independently in tool_used_status
-- Only extract factual content, do not add any additional explanations or information
+- A trajectory may involve multiple tools, each recorded independently in tool_used_status
+- If no tool was called, tool_used_status is an empty array []
+- If multiple trajectories have sequential dependencies, treat them as one trajectory
+- Only extract factual content, do not add any explanations or extra information
- Ensure the returned content is valid JSON format
+- The trajectory should be arranged according to the development order of messages
+- Experience must be general and reusable rules, not descriptions specific to concrete cases
+- Whether success or failed, always extract experience using when...then... format
+
+Please analyze the following conversation messages and extract tool call trajectories based on:
+
+{messages}
+
+"""
+
+
+TOOL_TRAJECTORY_PROMPT_ZH_BAK = """
+你是一个专业的工具经验提取专家。你的任务是从给定的对话消息中提取完整的工具调用轨迹经验。
-Please analyze the following conversation messages and extract tool call trajectories:
+## 分析判断步骤:
+**步骤1:判断任务完成度**
+根据用户反馈,判定correctness:success(成功)或 failed(失败),用户反馈决定权大于执行结果,用户反馈有误,则判定为failed
+**步骤2:成功轨迹(success)**
+记录轨迹信息,但经验留空,不需要记录
+
+**步骤3:失败轨迹(failed)- 错误分析与经验提炼**
+3.1 工具需求判断
+ - 任务是否需要工具?(需要/直接回答/误调用)
+3.2 工具调用检查
+ - 工具存在性:是否在system中提供
+ - 工具选择:是否选对工具
+ - 参数正确性:是否符合类型定义
+ - 幻觉检测:是否调用不存在的工具
+3.3 错误根因定位
+ 结合消息中的错误反馈信息和上述分析,精准输出根本原因
+3.4 经验提炼(核心)
+ 从失败模式中提炼通用原则或规则,采用"when...then..."结构:
+ - when: 明确描述触发该经验的场景特征(任务类型、工具环境、参数特征等)
+ - then: 给出避免错误的通用策略、正确调用方式或决策规则
+ 注意:经验是解决整个轨迹问题级别的,不仅仅针对单个工具
+
+## 输出格式:
+返回一个JSON数组,格式如下:
+
+```json
+[
+ {
+ "correctness": "success 或 failed",
+ "trajectory": "精炼完整的自然语言总结,包含:[任务(用户任务) -> 执行动作(调用的工具/直接回答) -> 执行结果] (可能多轮) -> 最终回答",
+ "experience": "如果成功:留空\n如果失败:采用when...then...格式,例如:'when 遇到XX的任务时,应该YY'",
+ "tool_used_status": [
+ {
+ "used_tool": "工具名称(如果调用了工具)",
+ "success_rate": "0.0-1.0之间的数值,表示该工具在本次轨迹中的成功率",
+ "error_type": "调用失败时的错误类型和描述,成功时为空字符串"
+ }
+ ]
+ }
+]
+```
+
+## 注意事项:
+- 每个轨迹必须是独立的完整过程
+- 一个轨迹中可能涉及多个工具的使用,每个工具在tool_used_status中独立记录
+- 如果没有调用工具,tool_used_status为空数组[]
+- 如果多条轨迹存在顺序依赖关系,需要将它们视为一条轨迹
+- 只提取事实内容,不要添加任何解释或额外信息
+- 确保返回的是有效的JSON格式
+- 输出的trajectory需要按照messages的发展顺序排列
+- 当任务完成度为success时,experience字段应留空(空字符串),但trajectory字段仍需记录完整轨迹
+- experience必须是通用的、可复用的经验规则,而不是针对具体案例的描述
+
+请分析以下对话消息并提取工具调用轨迹,基于以下对话消息:
+
{messages}
+
+"""
+
+
+TOOL_TRAJECTORY_PROMPT_EN_BAK = """
+You are a professional tool experience extraction expert. Your task is to extract complete tool call trajectory experiences from given conversation messages.
+
+## Analysis and Judgment Steps:
+**Step 1: Assess Task Completion**
+Determine correctness based on user feedback: success or failed, user feedback has higher priority than execution results, if user feedback is incorrect, then determine as failed
+
+**Step 2: Successful Trajectory (success)**
+Record trajectory information, but leave experience empty, no need to record
+
+**Step 3: Failed Trajectory (failed) - Error Analysis and Experience Extraction**
+
+3.1 Tool Requirement Assessment
+ - Does the task require tools? (required/direct answer/unnecessary call)
+
+3.2 Tool Call Verification
+ - Tool availability: provided in system?
+ - Tool selection: correct tool chosen?
+ - Parameter correctness: conform to type definitions?
+ - Hallucination detection: calling non-existent tools?
+
+3.3 Root Cause Identification
+ Combine error feedback from messages with above analysis to precisely output root cause
+
+3.4 Experience Extraction (Core)
+ Extract general principles or rules from failure patterns, using "when...then..." structure:
+ - when: clearly describe the scenario characteristics that trigger this experience (task type, tool environment, parameter characteristics, etc.)
+ - then: provide general strategies to avoid errors, correct calling approaches, or decision rules
+ Note: Experience is at the trajectory-level problem-solving, not just for a single tool
+
+## Output Format:
+Return a JSON array in the following format:
+
+```json
+[
+ {
+ "correctness": "success or failed",
+ "trajectory": "Concise and complete natural language summary including: [task (user task) -> execution action (tool called/direct answer) -> execution result] (possibly multiple rounds) -> final answer",
+ "experience": "If success: leave empty\nIf failed: use when...then... format, e.g., 'when encountering XX tasks, should do YY'",
+ "tool_used_status": [
+ {
+ "used_tool": "Tool name (if tool was called)",
+ "success_rate": "Numerical value between 0.0-1.0, indicating the success rate of this tool in current trajectory",
+ "error_type": "Error type and description when call fails, empty string when successful"
+ }
+ ]
+ }
+]
+```
+
+## Notes:
+- Each trajectory must be an independent complete process
+- A trajectory may involve multiple tools, each recorded independently in tool_used_status
+- If no tool was called, tool_used_status is an empty array []
+- If multiple trajectories have sequential dependencies, treat them as one trajectory
+- Only extract factual content, do not add any explanations or extra information
+- Ensure the returned content is valid JSON format
+- The trajectory should be arranged according to the development order of messages
+- When task completion is success, the experience field should be left empty (empty string), but the trajectory field should still record the complete trajectory
+- Experience must be general and reusable rules, not descriptions specific to concrete cases
+
+Please analyze the following conversation messages and extract tool call trajectories based on:
+
+{messages}
+
"""
diff --git a/src/memos/vec_dbs/milvus.py b/src/memos/vec_dbs/milvus.py
index 5dacf0499..cc8909d34 100644
--- a/src/memos/vec_dbs/milvus.py
+++ b/src/memos/vec_dbs/milvus.py
@@ -457,14 +457,13 @@ def get_by_id(self, collection_name: str, id: str) -> MilvusVecDBItem | None:
return None
entity = results[0]
- payload = {k: v for k, v in entity.items() if k not in ["id", "vector", "score"]}
return MilvusVecDBItem(
id=entity["id"],
memory=entity.get("memory"),
original_text=entity.get("original_text"),
vector=entity.get("vector"),
- payload=payload,
+ payload=entity.get("payload", {}),
)
def get_by_ids(self, collection_name: str, ids: list[str]) -> list[MilvusVecDBItem]:
@@ -479,14 +478,13 @@ def get_by_ids(self, collection_name: str, ids: list[str]) -> list[MilvusVecDBIt
items = []
for entity in results:
- payload = {k: v for k, v in entity.items() if k not in ["id", "vector", "score"]}
items.append(
MilvusVecDBItem(
id=entity["id"],
memory=entity.get("memory"),
original_text=entity.get("original_text"),
vector=entity.get("vector"),
- payload=payload,
+ payload=entity.get("payload", {}),
)
)