1616
1717from typing import Any
1818from typing import Optional
19+ from typing import Union
1920
2021from google .genai import types as genai_types
21- from pydantic import alias_generators
22- from pydantic import BaseModel
23- from pydantic import ConfigDict
2422from pydantic import Field
23+ from typing_extensions import TypeAlias
2524
25+ from .app_details import AppDetails
26+ from .common import EvalBaseModel
2627from .eval_rubrics import Rubric
2728
2829
29- class EvalBaseModel (BaseModel ):
30- model_config = ConfigDict (
31- alias_generator = alias_generators .to_camel ,
32- populate_by_name = True ,
33- )
34-
35-
3630class IntermediateData (EvalBaseModel ):
3731 """Container for intermediate data that an agent would generate as it responds with a final answer."""
3832
@@ -54,6 +48,33 @@ class IntermediateData(EvalBaseModel):
5448 """
5549
5650
51+ class InvocationEvent (EvalBaseModel ):
52+ """An immutable record representing a specific point in the agent's invocation.
53+
54+ It captures agent's replies, requests to use tools (function calls), and tool
55+ results.
56+
57+ This structure is a simple projection of the actual `Event` datamodel that
58+ is intended for the Eval System.
59+ """
60+
61+ author : str
62+ """The name of the agent that authored/owned this event."""
63+
64+ content : Optional [genai_types .Content ]
65+ """The content of the event."""
66+
67+
68+ class InvocationEvents (EvalBaseModel ):
69+ """A container for events that occur during the course of an invocation."""
70+
71+ invocation_events : list [InvocationEvent ] = Field (default_factory = list )
72+ """A list of invocation events."""
73+
74+
75+ IntermediateDataType : TypeAlias = Union [IntermediateData , InvocationEvents ]
76+
77+
5778class Invocation (EvalBaseModel ):
5879 """Represents a single invocation."""
5980
@@ -66,7 +87,7 @@ class Invocation(EvalBaseModel):
6687 final_response : Optional [genai_types .Content ] = None
6788 """Final response from the agent."""
6889
69- intermediate_data : Optional [IntermediateData ] = None
90+ intermediate_data : Optional [IntermediateDataType ] = None
7091 """Intermediate steps generated as a part of Agent execution.
7192
7293 For a multi-agent system, it is also helpful to inspect the route that
@@ -81,6 +102,9 @@ class Invocation(EvalBaseModel):
81102 )
82103 """A list of rubrics that are applicable to only this invocation."""
83104
105+ app_details : Optional [AppDetails ] = Field (default = None )
106+ """Details about the App that was used for this invocation."""
107+
84108
85109class SessionInput (EvalBaseModel ):
86110 """Values that help initialize a Session."""
@@ -117,3 +141,30 @@ class EvalCase(EvalBaseModel):
117141 default = None ,
118142 )
119143 """A list of rubrics that are applicable to all the invocations in the conversation of this eval case."""
144+
145+
146+ def get_all_tool_calls (
147+ intermediate_data : Optional [IntermediateDataType ],
148+ ) -> list [genai_types .FunctionCall ]:
149+ """A utility method to retrieve tools calls from intermediate data."""
150+ if not intermediate_data :
151+ return []
152+
153+ tool_calls = []
154+ if isinstance (intermediate_data , IntermediateData ):
155+ tool_calls = intermediate_data .tool_uses
156+ elif isinstance (intermediate_data , InvocationEvents ):
157+ # Go over each event in the list of events
158+ for invocation_event in intermediate_data .invocation_events :
159+ # Check if the event has content and some parts.
160+ if invocation_event .content and invocation_event .content .parts :
161+ for p in invocation_event .content .parts :
162+ # For each part, we check if any of those part is a function call.
163+ if p .function_call :
164+ tool_calls .append (p .function_call )
165+ else :
166+ raise ValueError (
167+ f"Unsupported type for intermediate_data `{ intermediate_data } `"
168+ )
169+
170+ return tool_calls
0 commit comments