-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Enhance tool usage tracking with metadata aggregation #4424
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -244,13 +244,24 @@ async def run_async( | |
| state=state_dict, | ||
| ) | ||
|
|
||
| # Aggregate usage metadata from sub-agent execution | ||
| usage_metadata_list = [] | ||
| last_content = None | ||
| async with Aclosing( | ||
| runner.run_async( | ||
| user_id=session.user_id, session_id=session.id, new_message=content | ||
| ) | ||
| ) as agen: | ||
| async for event in agen: | ||
| # Collect usage metadata from each event | ||
| if event.usage_metadata: | ||
| usage_metadata_list.append(event.usage_metadata) | ||
|
|
||
| # Also collect tool-level usage from nested events | ||
| if event.tool_usage_metadata: | ||
| for tool_name, tool_usage in event.tool_usage_metadata.items(): | ||
| usage_metadata_list.append(tool_usage) | ||
|
|
||
| # Forward state delta to parent session. | ||
| if event.actions.state_delta: | ||
| tool_context.state.update(event.actions.state_delta) | ||
|
|
@@ -261,6 +272,13 @@ async def run_async( | |
| # to avoid "Attempted to exit cancel scope in a different task" errors | ||
| await runner.close() | ||
|
|
||
| # Aggregate and record usage for this sub-agent | ||
| if usage_metadata_list: | ||
| from ..models.llm_response import LlmResponse | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| aggregated_usage = LlmResponse.merge_usage_metadata(usage_metadata_list) | ||
| if aggregated_usage: | ||
| tool_context.set_tool_usage(self.agent.name, aggregated_usage) | ||
|
|
||
| if last_content is None or last_content.parts is None: | ||
| return '' | ||
| merged_text = '\n'.join( | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -59,11 +59,45 @@ def __init__( | |||||
| super().__init__(invocation_context, event_actions=event_actions) | ||||||
| self.function_call_id = function_call_id | ||||||
| self.tool_confirmation = tool_confirmation | ||||||
| self._tool_usage: dict[str, Any] = {} | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For better type safety, consider using a more specific type hint instead of
Suggested change
|
||||||
|
|
||||||
| @property | ||||||
| def actions(self) -> EventActions: | ||||||
| return self._event_actions | ||||||
|
|
||||||
| def set_tool_usage( | ||||||
| self, | ||||||
| tool_name: str, | ||||||
| usage_metadata: Any, | ||||||
| ) -> None: | ||||||
| """Records usage metadata for a tool or sub-agent invocation. | ||||||
|
|
||||||
| Args: | ||||||
| tool_name: Name of the tool or agent that generated usage. | ||||||
| usage_metadata: Usage metadata object (GenerateContentResponseUsageMetadata | ||||||
| or dict with token counts). | ||||||
| """ | ||||||
| self._tool_usage[tool_name] = usage_metadata | ||||||
|
|
||||||
| def get_tool_usage(self, tool_name: str) -> Optional[Any]: | ||||||
| """Retrieves usage metadata for a specific tool. | ||||||
|
|
||||||
| Args: | ||||||
| tool_name: Name of the tool to retrieve usage for. | ||||||
|
|
||||||
| Returns: | ||||||
| Usage metadata if recorded, None otherwise. | ||||||
| """ | ||||||
| return self._tool_usage.get(tool_name) | ||||||
|
|
||||||
| def get_all_tool_usage(self) -> dict[str, Any]: | ||||||
| """Returns all tool usage metadata recorded in this context. | ||||||
|
|
||||||
| Returns: | ||||||
| Dictionary mapping tool names to their usage metadata. | ||||||
| """ | ||||||
| return self._tool_usage.copy() | ||||||
|
Comment on lines
+68
to
+99
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. To improve type safety and code clarity, the type hints for def set_tool_usage(
self,
tool_name: str,
usage_metadata: "types.GenerateContentResponseUsageMetadata",
) -> None:
"""Records usage metadata for a tool or sub-agent invocation.
Args:
tool_name: Name of the tool or agent that generated usage.
usage_metadata: Usage metadata object (GenerateContentResponseUsageMetadata
or dict with token counts).
"""
self._tool_usage[tool_name] = usage_metadata
def get_tool_usage(self, tool_name: str) -> Optional["types.GenerateContentResponseUsageMetadata"]:
"""Retrieves usage metadata for a specific tool.
Args:
tool_name: Name of the tool to retrieve usage for.
Returns:
Usage metadata if recorded, None otherwise.
"""
return self._tool_usage.get(tool_name)
def get_all_tool_usage(self) -> dict[str, "types.GenerateContentResponseUsageMetadata"]:
"""Returns all tool usage metadata recorded in this context.
Returns:
Dictionary mapping tool names to their usage metadata.
"""
return self._tool_usage.copy() |
||||||
|
|
||||||
| def request_credential(self, auth_config: AuthConfig) -> None: | ||||||
| if not self.function_call_id: | ||||||
| raise ValueError('function_call_id is not set.') | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The current implementation iterates over
function_response_event.tool_usage_metadatatwice: once to calculate the total token counts and again to create the breakdown dictionary. This can be optimized by performing both operations in a single loop. This refactoring will improve efficiency and make the code more concise.