Remove evaluator defs (#1255)

Author: akshaylive

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.7.0"
+version = "2.7.1"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

samples/calculator/evaluations/evaluators/custom/correct_operator.py

@@ -1,6 +1,7 @@
 import json
 
-from uipath.eval.evaluators import BaseEvaluator, BaseEvaluationCriteria, BaseEvaluatorConfig
+from uipath.eval.evaluators import BaseEvaluationCriteria, BaseEvaluatorConfig
+from uipath.eval.evaluators.base_evaluator import BaseEvaluator
 from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult
 from opentelemetry.sdk.trace import ReadableSpan
 

src/uipath/_cli/_evals/_console_progress_reporter.py

@@ -15,7 +15,7 @@
     EvalSetRunUpdatedEvent,
     EvaluationEvents,
 )
-from uipath.eval.evaluators import BaseEvaluator
+from uipath.eval.evaluators.base_evaluator import GenericBaseEvaluator
 from uipath.eval.models import ScoreType
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class ConsoleProgressReporter:
 
     def __init__(self):
         self.console = Console()
-        self.evaluators: dict[str, BaseEvaluator[Any, Any, Any]] = {}
+        self.evaluators: dict[str, GenericBaseEvaluator[Any, Any, Any]] = {}
         self.display_started = False
         self.eval_results_by_name: dict[str, list[Any]] = {}
 

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -9,85 +9,20 @@
 from uipath._cli._evals._helpers import (  # type: ignore # Remove after gnarly fix
     try_extract_file_and_class_name,
 )
-from uipath._cli._evals._models._evaluator import (
-    EvaluatorConfig,
-    LegacyEqualsEvaluatorParams,
-    LegacyEvaluator,
-    LegacyJsonSimilarityEvaluatorParams,
-    LegacyLLMEvaluatorParams,
-    LegacyTrajectoryEvaluatorParams,
-)
-from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
+from uipath._cli._evals._models._evaluator import CodedEvaluator, LegacyEvaluator
 from uipath._utils.constants import EVALS_FOLDER
 from uipath.eval.evaluators import (
     BaseEvaluator,
-    LegacyBaseEvaluator,
+    BaseLegacyEvaluator,
     LegacyContextPrecisionEvaluator,
-    LegacyExactMatchEvaluator,
     LegacyFaithfulnessEvaluator,
-    LegacyJsonSimilarityEvaluator,
     LegacyLlmAsAJudgeEvaluator,
     LegacyTrajectoryEvaluator,
 )
-from uipath.eval.evaluators.base_evaluator import BaseEvaluatorConfig
-from uipath.eval.evaluators.contains_evaluator import (
-    ContainsEvaluator,
-    ContainsEvaluatorConfig,
-)
-from uipath.eval.evaluators.exact_match_evaluator import (
-    ExactMatchEvaluator,
-    ExactMatchEvaluatorConfig,
-)
-from uipath.eval.evaluators.json_similarity_evaluator import (
-    JsonSimilarityEvaluator,
-    JsonSimilarityEvaluatorConfig,
-)
-from uipath.eval.evaluators.llm_judge_output_evaluator import (
-    LLMJudgeOutputEvaluator,
-    LLMJudgeOutputEvaluatorConfig,
-    LLMJudgeStrictJSONSimilarityOutputEvaluator,
-    LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig,
-)
-from uipath.eval.evaluators.llm_judge_trajectory_evaluator import (
-    LLMJudgeTrajectoryEvaluator,
-    LLMJudgeTrajectoryEvaluatorConfig,
-    LLMJudgeTrajectorySimulationEvaluator,
-    LLMJudgeTrajectorySimulationEvaluatorConfig,
-)
-from uipath.eval.evaluators.tool_call_args_evaluator import (
-    ToolCallArgsEvaluator,
-    ToolCallArgsEvaluatorConfig,
-)
-from uipath.eval.evaluators.tool_call_count_evaluator import (
-    ToolCallCountEvaluator,
-    ToolCallCountEvaluatorConfig,
-)
-from uipath.eval.evaluators.tool_call_order_evaluator import (
-    ToolCallOrderEvaluator,
-    ToolCallOrderEvaluatorConfig,
-)
-from uipath.eval.evaluators.tool_call_output_evaluator import (
-    ToolCallOutputEvaluator,
-    ToolCallOutputEvaluatorConfig,
-)
-from uipath.eval.models import LegacyEvaluatorType
+from uipath.eval.evaluators.base_evaluator import GenericBaseEvaluator
 
 logger = logging.getLogger(__name__)
 
-EVALUATOR_SCHEMA_TO_EVALUATOR_CLASS = {
-    ContainsEvaluatorConfig: ContainsEvaluator,
-    ExactMatchEvaluatorConfig: ExactMatchEvaluator,
-    JsonSimilarityEvaluatorConfig: JsonSimilarityEvaluator,
-    LLMJudgeOutputEvaluatorConfig: LLMJudgeOutputEvaluator,
-    LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig: LLMJudgeStrictJSONSimilarityOutputEvaluator,
-    LLMJudgeTrajectoryEvaluatorConfig: LLMJudgeTrajectoryEvaluator,
-    LLMJudgeTrajectorySimulationEvaluatorConfig: LLMJudgeTrajectorySimulationEvaluator,
-    ToolCallArgsEvaluatorConfig: ToolCallArgsEvaluator,
-    ToolCallCountEvaluatorConfig: ToolCallCountEvaluator,
-    ToolCallOrderEvaluatorConfig: ToolCallOrderEvaluator,
-    ToolCallOutputEvaluatorConfig: ToolCallOutputEvaluator,
-}
-
 
 class EvaluatorFactory:
     """Factory class for creating evaluator instances based on configuration."""
@@ -130,7 +65,7 @@ def create_evaluator(
         data: dict[str, Any],
         evaluators_dir: Path | None = None,
         agent_model: str | None = None,
-    ) -> BaseEvaluator[Any, Any, Any]:
+    ) -> GenericBaseEvaluator[Any, Any, Any]:
         if data.get("version", None) == "1.0":
             return cls._create_evaluator_internal(data, evaluators_dir)
         else:
@@ -147,31 +82,20 @@ def _create_evaluator_internal(
             evaluator_schema
         )
         if success:
-            return EvaluatorFactory._create_coded_evaluator_internal(
+            return EvaluatorFactory._create_custom_coded_evaluator_internal(
                 data, file_path, class_name, evaluators_dir
             )
-
-        config: BaseEvaluatorConfig[Any] = TypeAdapter(EvaluatorConfig).validate_python(
-            data
-        )
-        evaluator_class = EVALUATOR_SCHEMA_TO_EVALUATOR_CLASS.get(type(config))
-        if not evaluator_class:
-            raise ValueError(f"Unknown evaluator configuration: {config}")
-        return TypeAdapter(evaluator_class).validate_python(
-            {
-                "id": data.get("id"),
-                "config": EvaluatorFactory._prepare_evaluator_config(data),
-            }
-        )
+        else:
+            return TypeAdapter(CodedEvaluator).validate_python(data)
 
     @staticmethod
-    def _create_coded_evaluator_internal(
+    def _create_custom_coded_evaluator_internal(
         data: dict[str, Any],
         file_path_str: str,
         class_name: str,
         evaluators_dir: Path | None = None,
     ) -> BaseEvaluator[Any, Any, Any]:
-        """Create a coded evaluator by dynamically loading from a Python file.
+        """Create a custom coded evaluator by dynamically loading from a Python file.
 
         Args:
             data: Dictionary containing evaluator configuration with evaluatorTypeId
@@ -242,18 +166,13 @@ def _create_coded_evaluator_internal(
         evaluator_id = data.get("id")
         if not evaluator_id or not isinstance(evaluator_id, str):
             raise ValueError("Evaluator 'id' must be a non-empty string")
-        return TypeAdapter(evaluator_class).validate_python(
-            {
-                "id": evaluator_id,
-                "config": EvaluatorFactory._prepare_evaluator_config(data),
-            }
-        )
+        return TypeAdapter(evaluator_class).validate_python(data)
 
     @staticmethod
     def _create_legacy_evaluator_internal(
         data: dict[str, Any],
         agent_model: str | None = None,
-    ) -> LegacyBaseEvaluator[Any]:
+    ) -> BaseLegacyEvaluator[Any]:
         """Create an evaluator instance from configuration data.
 
         Args:
@@ -267,97 +186,25 @@ def _create_legacy_evaluator_internal(
         Raises:
             ValueError: If category is unknown or required fields are missing
         """
-        params: EvaluatorBaseParams = TypeAdapter(LegacyEvaluator).validate_python(data)
-
-        match params:
-            case LegacyEqualsEvaluatorParams():
-                return EvaluatorFactory._create_legacy_exact_match_evaluator(params)
-            case LegacyJsonSimilarityEvaluatorParams():
-                return EvaluatorFactory._create_legacy_json_similarity_evaluator(params)
-            case LegacyLLMEvaluatorParams():
-                return EvaluatorFactory._create_legacy_llm_as_judge_evaluator(
-                    params, agent_model
-                )
-            case LegacyTrajectoryEvaluatorParams():
-                return EvaluatorFactory._create_legacy_trajectory_evaluator(
-                    params, agent_model
-                )
-            case _:
-                raise ValueError(f"Unknown evaluator category: {params}")
-
-    @staticmethod
-    def _create_legacy_exact_match_evaluator(
-        params: LegacyEqualsEvaluatorParams,
-    ) -> LegacyExactMatchEvaluator:
-        """Create a deterministic evaluator."""
-        return LegacyExactMatchEvaluator(**params.model_dump(), config={})
-
-    @staticmethod
-    def _create_legacy_json_similarity_evaluator(
-        params: LegacyJsonSimilarityEvaluatorParams,
-    ) -> LegacyJsonSimilarityEvaluator:
-        """Create a deterministic evaluator."""
-        return LegacyJsonSimilarityEvaluator(**params.model_dump(), config={})
-
-    @staticmethod
-    def _create_legacy_llm_as_judge_evaluator(
-        params: LegacyLLMEvaluatorParams,
-        agent_model: str | None = None,
-    ) -> LegacyBaseEvaluator[Any]:
-        """Create an LLM-as-a-judge evaluator or context precision evaluator based on type."""
-        if not params.model:
-            raise ValueError("LLM evaluator must include 'model' field")
-
-        # Resolve 'same-as-agent' to actual agent model
-        if params.model == "same-as-agent":
-            if not agent_model:
-                raise ValueError(
-                    "'same-as-agent' model option requires agent settings. "
-                    "Ensure agent.json contains valid model settings."
-                )
-            logger.info(
-                f"Resolving 'same-as-agent' to agent model: {agent_model} "
-                f"for evaluator '{params.name}'"
-            )
-            params = params.model_copy(update={"model": agent_model})
-
-        # Check evaluator type to determine which evaluator to create
-        if params.evaluator_type == LegacyEvaluatorType.ContextPrecision:
-            return LegacyContextPrecisionEvaluator(**params.model_dump(), config={})
-        elif params.evaluator_type == LegacyEvaluatorType.Faithfulness:
-            return LegacyFaithfulnessEvaluator(**params.model_dump(), config={})
-        else:
-            if not params.prompt:
-                raise ValueError("LLM evaluator must include 'prompt' field")
-
-            return LegacyLlmAsAJudgeEvaluator(**params.model_dump(), config={})
-
-    @staticmethod
-    def _create_legacy_trajectory_evaluator(
-        params: LegacyTrajectoryEvaluatorParams,
-        agent_model: str | None = None,
-    ) -> LegacyTrajectoryEvaluator:
-        """Create a trajectory evaluator."""
-        if not params.prompt:
-            raise ValueError("Trajectory evaluator must include 'prompt' field")
-
-        if not params.model:
-            raise ValueError("Trajectory evaluator must include 'model' field")
-
-        # Resolve 'same-as-agent' to actual agent model
-        if params.model == "same-as-agent":
-            if not agent_model:
-                raise ValueError(
-                    "'same-as-agent' model option requires agent settings. "
-                    "Ensure agent.json contains valid model settings."
+        evaluator: LegacyEvaluator = TypeAdapter(LegacyEvaluator).validate_python(data)
+
+        if isinstance(
+            evaluator,
+            LegacyTrajectoryEvaluator
+            | LegacyLlmAsAJudgeEvaluator
+            | LegacyContextPrecisionEvaluator
+            | LegacyFaithfulnessEvaluator,
+        ):
+            if evaluator.model == "same-as-agent":
+                if not agent_model:
+                    raise ValueError(
+                        "'same-as-agent' model option requires agent settings. "
+                        "Ensure agent.json contains valid model settings."
+                    )
+                logger.info(
+                    f"Resolving 'same-as-agent' to agent model: {agent_model} "
+                    f"for evaluator '{evaluator.name}'"
                 )
-            logger.info(
-                f"Resolving 'same-as-agent' to agent model: {agent_model} "
-                f"for evaluator '{params.name}'"
-            )
-            params = params.model_copy(update={"model": agent_model})
+                evaluator.model = agent_model
 
-        logger.info(
-            f"Creating trajectory evaluator '{params.name}' with model: {params.model}"
-        )
-        return LegacyTrajectoryEvaluator(**params.model_dump(), config={})
+        return evaluator