diff --git a/tests/unit/vertexai/genai/replays/test_create_evaluation_run.py b/tests/unit/vertexai/genai/replays/test_create_evaluation_run.py index 3fb251d16d..91e70afb3e 100644 --- a/tests/unit/vertexai/genai/replays/test_create_evaluation_run.py +++ b/tests/unit/vertexai/genai/replays/test_create_evaluation_run.py @@ -683,6 +683,35 @@ def test_create_eval_run_with_metric_resource_name(mock_uuid4, client): # == INPUT_DF_WITH_CONTEXT_AND_HISTORY.iloc[i]["response"] # ) # assert evaluation_run.error is None +def test_create_eval_run_with_red_teaming_config(client): + """Tests that create_evaluation_run() with red_teaming_config sends analysisConfigs.""" + evaluation_run = client.evals.create_evaluation_run( + name="test_red_teaming", + display_name="test_red_teaming", + dataset=types.EvaluationRunDataSource(evaluation_set=EVAL_SET_NAME), + dest=GCS_DEST, + metrics=[], + red_teaming_config=types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + vulnerable_tools=[ + types.VulnerableTool( + tool_name="search_flights", + json_paths=["$.flights[0].description"], + ), + ], + ), + ) + assert isinstance(evaluation_run, types.EvaluationRun) + assert evaluation_run.display_name == "test_red_teaming" + assert evaluation_run.state == types.EvaluationRunState.PENDING + assert evaluation_run.analysis_configs is not None + assert len(evaluation_run.analysis_configs) == 1 + rt_config = evaluation_run.analysis_configs[0].red_teaming_analysis_config + assert rt_config.attack_categories == ["FINANCIAL_OR_CREDENTIAL_PHISHING"] + assert rt_config.vulnerable_tools[0].tool_name == "search_flights" + assert evaluation_run.error is None + + pytest_plugins = ("pytest_asyncio",) diff --git a/tests/unit/vertexai/genai/test_evals.py b/tests/unit/vertexai/genai/test_evals.py index 1f15ac95a3..5ebb6c35ba 100644 --- a/tests/unit/vertexai/genai/test_evals.py +++ b/tests/unit/vertexai/genai/test_evals.py @@ -1835,6 +1835,146 @@ def test_loss_analysis_metrics_accepts_metric_object(self): assert result[0].candidate == "agent-1" +class TestRedTeamingTypes: + """Unit tests for red teaming type definitions.""" + + def test_red_teaming_analysis_config_construction(self): + config = common_types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + vulnerable_tools=[ + common_types.VulnerableTool( + tool_name="search_flights", + json_paths=["$.flights[0].description"], + ), + ], + ) + assert len(config.attack_categories) == 1 + assert config.vulnerable_tools[0].tool_name == "search_flights" + + def test_red_teaming_analysis_config_optional_fields(self): + config = common_types.RedTeamingAnalysisConfig() + assert config.attack_categories is None + assert config.vulnerable_tools is None + + def test_evaluation_run_results_has_red_teaming_results(self): + results = common_types.EvaluationRunResults( + red_teaming_analysis_results=[ + common_types.RedTeamingAnalysisResult( + category_results=[ + common_types.AttackCategoryResult( + attack_category="FINANCIAL_OR_CREDENTIAL_PHISHING", + attack_success_rate=0.9, + ), + ], + ) + ], + ) + assert len(results.red_teaming_analysis_results) == 1 + assert ( + results.red_teaming_analysis_results[0] + .category_results[0] + .attack_success_rate + == 0.9 + ) + + def test_create_params_accepts_analysis_configs(self): + params = common_types._CreateEvaluationRunParameters( + name="test-run", + analysis_configs=[ + common_types.AnalysisConfig( + red_teaming_analysis_config=common_types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + ), + ), + ], + ) + assert len(params.analysis_configs) == 1 + + +class TestResolveRedTeamingConfig: + """Unit tests for _resolve_red_teaming_config.""" + + def test_none_when_no_config(self): + result = _evals_utils._resolve_red_teaming_config() + assert result is None + + def test_wraps_config_in_analysis_configs(self): + config = common_types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + ) + result = _evals_utils._resolve_red_teaming_config(config) + assert len(result) == 1 + assert isinstance(result[0], common_types.AnalysisConfig) + assert ( + result[0].red_teaming_analysis_config.attack_categories[0] + == "FINANCIAL_OR_CREDENTIAL_PHISHING" + ) + + def test_accepts_dict_input(self): + result = _evals_utils._resolve_red_teaming_config( + {"attack_categories": ["INJECTED_HOSTILITY_AND_HARASSMENT"]} + ) + assert len(result) == 1 + assert isinstance(result[0], common_types.AnalysisConfig) + + +class TestRedTeamingSerializationConverters: + """Unit tests for red teaming serialization converters.""" + + def test_analysis_config_to_vertex(self): + config = common_types.AnalysisConfig( + analysis_name="my-analysis", + red_teaming_analysis_config=common_types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + vulnerable_tools=[ + common_types.VulnerableTool( + tool_name="search_flights", + json_paths=["$.flights[0].description"], + ), + ], + ), + ) + result = evals._AnalysisConfig_to_vertex(config) + assert result["analysisName"] == "my-analysis" + rt = result["redTeamingAnalysisConfig"] + assert rt["attackCategories"] == ["FINANCIAL_OR_CREDENTIAL_PHISHING"] + assert rt["vulnerableTools"][0]["toolName"] == "search_flights" + + def test_analysis_config_from_vertex(self): + api_response = { + "analysisName": "my-analysis", + "redTeamingAnalysisConfig": { + "attackCategories": ["FINANCIAL_OR_CREDENTIAL_PHISHING"], + "vulnerableTools": [ + {"toolName": "search_flights", "jsonPaths": ["$.flights[0].description"]}, + ], + }, + } + result = evals._AnalysisConfig_from_vertex(api_response) + assert result["analysis_name"] == "my-analysis" + rt = result["red_teaming_analysis_config"] + assert rt["attack_categories"] == ["FINANCIAL_OR_CREDENTIAL_PHISHING"] + assert rt["vulnerable_tools"][0]["tool_name"] == "search_flights" + + def test_analysis_config_round_trip(self): + original = common_types.AnalysisConfig( + analysis_name="round-trip", + red_teaming_analysis_config=common_types.RedTeamingAnalysisConfig( + attack_categories=["FINANCIAL_OR_CREDENTIAL_PHISHING"], + vulnerable_tools=[ + common_types.VulnerableTool(tool_name="search_flights"), + ], + ), + ) + serialized = evals._AnalysisConfig_to_vertex(original) + deserialized = evals._AnalysisConfig_from_vertex(serialized) + assert deserialized["analysis_name"] == "round-trip" + assert ( + deserialized["red_teaming_analysis_config"]["vulnerable_tools"][0]["tool_name"] + == "search_flights" + ) + + class TestResolveMetricName: """Unit tests for _resolve_metric_name.""" diff --git a/vertexai/_genai/_evals_utils.py b/vertexai/_genai/_evals_utils.py index feb24bfbf1..97bc2d478d 100644 --- a/vertexai/_genai/_evals_utils.py +++ b/vertexai/_genai/_evals_utils.py @@ -541,6 +541,20 @@ def _resolve_eval_run_loss_configs( return configs +def _resolve_red_teaming_config( + red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None, +) -> Optional[list[types.AnalysisConfig]]: + """Wraps a RedTeamingAnalysisConfig into analysis_configs for the API.""" + if not red_teaming_config: + return None + config = ( + types.RedTeamingAnalysisConfig.model_validate(red_teaming_config) + if isinstance(red_teaming_config, dict) + else red_teaming_config + ) + return [types.AnalysisConfig(red_teaming_analysis_config=config)] + + def _resolve_loss_analysis_config( eval_result: types.EvaluationResult, config: Optional[types.LossAnalysisConfig] = None, diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 2e455f116b..2e14f5b65c 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -127,6 +127,16 @@ def _CreateEvaluationRunParameters_to_vertex( }, ) + if getv(from_object, ["analysis_configs"]) is not None: + setv( + to_object, + ["analysisConfigs"], + [ + _AnalysisConfig_to_vertex(item, to_object) + for item in getv(from_object, ["analysis_configs"]) + ], + ) + if getv(from_object, ["config"]) is not None: setv(to_object, ["config"], getv(from_object, ["config"])) @@ -428,6 +438,80 @@ def _EvaluationRunConfig_to_vertex( return to_object +def _AnalysisConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["analysis_name"]) is not None: + setv(to_object, ["analysisName"], getv(from_object, ["analysis_name"])) + if getv(from_object, ["red_teaming_analysis_config"]) is not None: + rt_config = getv(from_object, ["red_teaming_analysis_config"]) + rt_to: dict[str, Any] = {} + if getv(rt_config, ["attack_categories"]) is not None: + setv(rt_to, ["attackCategories"], getv(rt_config, ["attack_categories"])) + if getv(rt_config, ["vulnerable_tools"]) is not None: + setv( + rt_to, + ["vulnerableTools"], + [ + _VulnerableTool_to_vertex(tool, rt_to) + for tool in getv(rt_config, ["vulnerable_tools"]) + ], + ) + setv(to_object, ["redTeamingAnalysisConfig"], rt_to) + return to_object + + +def _VulnerableTool_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["tool_name"]) is not None: + setv(to_object, ["toolName"], getv(from_object, ["tool_name"])) + if getv(from_object, ["json_paths"]) is not None: + setv(to_object, ["jsonPaths"], getv(from_object, ["json_paths"])) + return to_object + + +def _AnalysisConfig_from_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["analysisName"]) is not None: + setv(to_object, ["analysis_name"], getv(from_object, ["analysisName"])) + if getv(from_object, ["redTeamingAnalysisConfig"]) is not None: + rt_config = getv(from_object, ["redTeamingAnalysisConfig"]) + rt_to: dict[str, Any] = {} + if getv(rt_config, ["attackCategories"]) is not None: + setv(rt_to, ["attack_categories"], getv(rt_config, ["attackCategories"])) + if getv(rt_config, ["vulnerableTools"]) is not None: + setv( + rt_to, + ["vulnerable_tools"], + [ + _VulnerableTool_from_vertex(tool, rt_to) + for tool in getv(rt_config, ["vulnerableTools"]) + ], + ) + setv(to_object, ["red_teaming_analysis_config"], rt_to) + return to_object + + +def _VulnerableTool_from_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["toolName"]) is not None: + setv(to_object, ["tool_name"], getv(from_object, ["toolName"])) + if getv(from_object, ["jsonPaths"]) is not None: + setv(to_object, ["json_paths"], getv(from_object, ["jsonPaths"])) + return to_object + + def _EvaluationRunInferenceConfig_from_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -589,6 +673,16 @@ def _EvaluationRun_from_vertex( if getv(from_object, ["labels"]) is not None: setv(to_object, ["labels"], getv(from_object, ["labels"])) + if getv(from_object, ["analysisConfigs"]) is not None: + setv( + to_object, + ["analysis_configs"], + [ + _AnalysisConfig_from_vertex(item, to_object) + for item in getv(from_object, ["analysisConfigs"]) + ], + ) + return to_object @@ -1144,6 +1238,7 @@ def _create_evaluation_run( inference_configs: Optional[ dict[str, types.EvaluationRunInferenceConfigOrDict] ] = None, + analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, ) -> types.EvaluationRun: """ @@ -1157,6 +1252,7 @@ def _create_evaluation_run( evaluation_config=evaluation_config, labels=labels, inference_configs=inference_configs, + analysis_configs=analysis_configs, config=config, ) @@ -2602,6 +2698,7 @@ def create_evaluation_run( labels: Optional[dict[str, str]] = None, loss_analysis_metrics: Optional[list[Union[str, types.MetricOrDict]]] = None, loss_analysis_configs: Optional[list[types.LossAnalysisConfigOrDict]] = None, + red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, ) -> types.EvaluationRun: """Creates an EvaluationRun. @@ -2708,6 +2805,9 @@ def create_evaluation_run( loss_analysis_configs=loss_analysis_configs, inference_configs=inference_configs, ) + resolved_analysis_configs = _evals_utils._resolve_red_teaming_config( + red_teaming_config + ) evaluation_config = types.EvaluationRunConfig( output_config=output_config, metrics=resolved_metrics, @@ -2724,6 +2824,7 @@ def create_evaluation_run( data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, + analysis_configs=resolved_analysis_configs, labels=resolved_labels, config=config, ) @@ -3271,6 +3372,7 @@ async def _create_evaluation_run( inference_configs: Optional[ dict[str, types.EvaluationRunInferenceConfigOrDict] ] = None, + analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, ) -> types.EvaluationRun: """ @@ -3284,6 +3386,7 @@ async def _create_evaluation_run( evaluation_config=evaluation_config, labels=labels, inference_configs=inference_configs, + analysis_configs=analysis_configs, config=config, ) @@ -4368,6 +4471,7 @@ async def create_evaluation_run( inference_configs: Optional[ dict[str, types.EvaluationRunInferenceConfigOrDict] ] = None, + red_teaming_config: Optional[types.RedTeamingAnalysisConfigOrDict] = None, labels: Optional[dict[str, str]] = None, loss_analysis_metrics: Optional[list[Union[str, types.MetricOrDict]]] = None, loss_analysis_configs: Optional[list[types.LossAnalysisConfigOrDict]] = None, @@ -4477,6 +4581,9 @@ async def create_evaluation_run( loss_analysis_configs=loss_analysis_configs, inference_configs=inference_configs, ) + resolved_analysis_configs = _evals_utils._resolve_red_teaming_config( + red_teaming_config + ) evaluation_config = types.EvaluationRunConfig( output_config=output_config, metrics=resolved_metrics, @@ -4494,6 +4601,7 @@ async def create_evaluation_run( data_source=resolved_dataset, evaluation_config=evaluation_config, inference_configs=resolved_inference_configs, + analysis_configs=resolved_analysis_configs, labels=resolved_labels, config=config, ) diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index 8ed8e6df4a..8257655087 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -718,6 +718,12 @@ from .common import ListSandboxEnvironmentTemplatesResponseDict from .common import ListSandboxEnvironmentTemplatesResponseOrDict from .common import LLMMetric +from .common import AnalysisConfig +from .common import AnalysisConfigDict +from .common import AnalysisConfigOrDict +from .common import AttackCategoryResult +from .common import AttackCategoryResultDict +from .common import AttackCategoryResultOrDict from .common import LossAnalysisConfig from .common import LossAnalysisConfigDict from .common import LossAnalysisConfigOrDict @@ -1024,6 +1030,12 @@ from .common import ReasoningEngineTrafficConfigTrafficSplitManualTarget from .common import ReasoningEngineTrafficConfigTrafficSplitManualTargetDict from .common import ReasoningEngineTrafficConfigTrafficSplitManualTargetOrDict +from .common import RedTeamingAnalysisConfig +from .common import RedTeamingAnalysisConfigDict +from .common import RedTeamingAnalysisConfigOrDict +from .common import RedTeamingAnalysisResult +from .common import RedTeamingAnalysisResultDict +from .common import RedTeamingAnalysisResultOrDict from .common import ReservationAffinity from .common import ReservationAffinityDict from .common import ReservationAffinityOrDict @@ -1378,6 +1390,9 @@ from .common import VertexBaseConfig from .common import VertexBaseConfigDict from .common import VertexBaseConfigOrDict +from .common import VulnerableTool +from .common import VulnerableToolDict +from .common import VulnerableToolOrDict from .common import WinRateStats from .common import WinRateStatsDict from .common import WinRateStatsOrDict @@ -1500,6 +1515,15 @@ "EvaluationRunPromptTemplate", "EvaluationRunPromptTemplateDict", "EvaluationRunPromptTemplateOrDict", + "VulnerableTool", + "VulnerableToolDict", + "VulnerableToolOrDict", + "RedTeamingAnalysisConfig", + "RedTeamingAnalysisConfigDict", + "RedTeamingAnalysisConfigOrDict", + "AnalysisConfig", + "AnalysisConfigDict", + "AnalysisConfigOrDict", "LossAnalysisConfig", "LossAnalysisConfigDict", "LossAnalysisConfigOrDict", @@ -1536,6 +1560,12 @@ "LossAnalysisResult", "LossAnalysisResultDict", "LossAnalysisResultOrDict", + "AttackCategoryResult", + "AttackCategoryResultDict", + "AttackCategoryResultOrDict", + "RedTeamingAnalysisResult", + "RedTeamingAnalysisResultDict", + "RedTeamingAnalysisResultOrDict", "EvaluationRunResults", "EvaluationRunResultsDict", "EvaluationRunResultsOrDict", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 82fe71fe98..3d5e1040f2 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2366,6 +2366,80 @@ class LossAnalysisConfigDict(TypedDict, total=False): LossAnalysisConfigOrDict = Union[LossAnalysisConfig, LossAnalysisConfigDict] +class VulnerableTool(_common.BaseModel): + """A tool considered high risk for prompt injection.""" + + tool_name: Optional[str] = Field( + default=None, + description="""Required. The exact name of the vulnerable function/tool (e.g., "search_flights").""", + ) + json_paths: Optional[list[str]] = Field( + default=None, + description="""Optional. JSON Paths within the tool's FunctionResponse where malicious content could be injected.""", + ) + + +class VulnerableToolDict(TypedDict, total=False): + """A tool considered high risk for prompt injection.""" + + tool_name: Optional[str] + + json_paths: Optional[list[str]] + + +VulnerableToolOrDict = Union[VulnerableTool, VulnerableToolDict] + + +class RedTeamingAnalysisConfig(_common.BaseModel): + """Configuration for the automated Agent Red Teaming analysis.""" + + attack_categories: Optional[list[str]] = Field( + default=None, + description="""Optional. Specific attack categories to test against.""", + ) + vulnerable_tools: Optional[list[VulnerableTool]] = Field( + default=None, + description="""Optional. Manually defined vulnerable tools and their injection paths.""", + ) + + +class RedTeamingAnalysisConfigDict(TypedDict, total=False): + """Configuration for the automated Agent Red Teaming analysis.""" + + attack_categories: Optional[list[str]] + + vulnerable_tools: Optional[list[VulnerableToolDict]] + + +RedTeamingAnalysisConfigOrDict = Union[ + RedTeamingAnalysisConfig, RedTeamingAnalysisConfigDict +] + + +class AnalysisConfig(_common.BaseModel): + """Configuration for an analysis to be performed on an evaluation run.""" + + analysis_name: Optional[str] = Field( + default=None, + description="""Optional. A name for this analysis.""", + ) + red_teaming_analysis_config: Optional[RedTeamingAnalysisConfig] = Field( + default=None, + description="""Configuration for the automated Agent Red Teaming analysis.""", + ) + + +class AnalysisConfigDict(TypedDict, total=False): + """Configuration for an analysis to be performed on an evaluation run.""" + + analysis_name: Optional[str] + + red_teaming_analysis_config: Optional[RedTeamingAnalysisConfigDict] + + +AnalysisConfigOrDict = Union[AnalysisConfig, AnalysisConfigDict] + + class EvaluationRunConfig(_common.BaseModel): """The evaluation configuration used for the evaluation run.""" @@ -2565,6 +2639,9 @@ class _CreateEvaluationRunParameters(_common.BaseModel): inference_configs: Optional[dict[str, EvaluationRunInferenceConfig]] = Field( default=None, description="""""" ) + analysis_configs: Optional[list[AnalysisConfig]] = Field( + default=None, description="""""" + ) config: Optional[CreateEvaluationRunConfig] = Field( default=None, description="""""" ) @@ -2591,6 +2668,9 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): inference_configs: Optional[dict[str, EvaluationRunInferenceConfigDict]] """""" + analysis_configs: Optional[list[AnalysisConfigDict]] + """""" + config: Optional[CreateEvaluationRunConfigDict] """""" @@ -2799,6 +2879,70 @@ class LossAnalysisResultDict(TypedDict, total=False): LossAnalysisResultOrDict = Union[LossAnalysisResult, LossAnalysisResultDict] +class AttackCategoryResult(_common.BaseModel): + """The red teaming outcome for a specific attack category.""" + + attack_category: Optional[str] = Field( + default=None, + description="""The category of the attack evaluated.""", + ) + attack_success_rate: Optional[float] = Field( + default=None, + description="""The ratio of successful attacks given a fixed budget.""", + ) + vulnerability_insight: Optional[str] = Field( + default=None, + description="""Insights into why an attack succeeded or failed.""", + ) + + +class AttackCategoryResultDict(TypedDict, total=False): + """The red teaming outcome for a specific attack category.""" + + attack_category: Optional[str] + + attack_success_rate: Optional[float] + + vulnerability_insight: Optional[str] + + +AttackCategoryResultOrDict = Union[ + AttackCategoryResult, AttackCategoryResultDict +] + + +class RedTeamingAnalysisResult(_common.BaseModel): + """The top-level result for Red Teaming analysis.""" + + config: Optional[RedTeamingAnalysisConfig] = Field( + default=None, + description="""The configuration used to generate this analysis.""", + ) + analysis_time: Optional[str] = Field( + default=None, + description="""The timestamp when this analysis was performed.""", + ) + category_results: Optional[list[AttackCategoryResult]] = Field( + default=None, + description="""Detailed results by attack category.""", + ) + + +class RedTeamingAnalysisResultDict(TypedDict, total=False): + """The top-level result for Red Teaming analysis.""" + + config: Optional[RedTeamingAnalysisConfigDict] + + analysis_time: Optional[str] + + category_results: Optional[list[AttackCategoryResultDict]] + + +RedTeamingAnalysisResultOrDict = Union[ + RedTeamingAnalysisResult, RedTeamingAnalysisResultDict +] + + class EvaluationRunResults(_common.BaseModel): """Represents the results of an evaluation run.""" @@ -2813,6 +2957,12 @@ class EvaluationRunResults(_common.BaseModel): default=None, description="""The loss analysis results for the evaluation run.""", ) + red_teaming_analysis_results: Optional[ + list[RedTeamingAnalysisResult] + ] = Field( + default=None, + description="""The Red Teaming analysis results.""", + ) class EvaluationRunResultsDict(TypedDict, total=False): @@ -2827,6 +2977,11 @@ class EvaluationRunResultsDict(TypedDict, total=False): loss_analysis_results: Optional[list[LossAnalysisResultDict]] """The loss analysis results for the evaluation run.""" + red_teaming_analysis_results: Optional[ + list[RedTeamingAnalysisResultDict] + ] + """The Red Teaming analysis results.""" + EvaluationRunResultsOrDict = Union[EvaluationRunResults, EvaluationRunResultsDict] @@ -3371,6 +3526,10 @@ class EvaluationRun(_common.BaseModel): description="""This field is experimental and may change in future versions. The inference configs for the evaluation run.""", ) labels: Optional[dict[str, str]] = Field(default=None, description="""""") + analysis_configs: Optional[list[AnalysisConfig]] = Field( + default=None, + description="""The analysis configurations for the evaluation run.""", + ) # TODO(b/448806531): Remove all the overridden _from_response methods once the # ticket is resolved and published.