Allow variant configuration to be passed dynamically @ inference (tensorzero#2931)

* refactored InferenceConfig & sample_variant

* wip

* factored preparation of variants into a helper & added parameter to inference handler

* everything but conversion is implemented

* fixed container build thing

* wip

* huh?

* a file

* wip

* compiler errors pass

* removed some unncessary Cows and added typescript derives

* check for dryrun

* added e2e tests for dynamic variant

* fixed issues with merge

* built bindings

* updated name of dynamic variant

* improved test

* fixed test

* fixed bad e2e test

Author: virajmehta

clients/python/src/lib.rs

@@ -498,6 +498,7 @@ impl BaseTensorZeroGateway {
             include_original_response,
             extra_body,
             extra_headers,
+            internal_dynamic_variant_config: None,
         })
     }
 }

clients/rust/src/client_inference_params.rs

@@ -5,11 +5,13 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use tensorzero_core::{
     cache::CacheParamsOptions,
+    config_parser::UninitializedVariantInfo,
     endpoints::inference::{InferenceParams, Params},
     error::Error,
-    inference::types::extra_body::UnfilteredInferenceExtraBody,
-    inference::types::extra_headers::UnfilteredInferenceExtraHeaders,
-    inference::types::{Input, InputMessage, InputMessageContent},
+    inference::types::{
+        extra_body::UnfilteredInferenceExtraBody, extra_headers::UnfilteredInferenceExtraHeaders,
+        Input, InputMessage, InputMessageContent,
+    },
     tool::DynamicToolParams,
 };
 use uuid::Uuid;
@@ -70,6 +72,7 @@ pub struct ClientInferenceParams {
     pub extra_body: UnfilteredInferenceExtraBody,
     #[serde(default)]
     pub extra_headers: UnfilteredInferenceExtraHeaders,
+    pub internal_dynamic_variant_config: Option<UninitializedVariantInfo>,
 }
 
 impl TryFrom<ClientInferenceParams> for Params {
@@ -112,6 +115,7 @@ impl TryFrom<ClientInferenceParams> for Params {
             include_original_response: this.include_original_response,
             extra_body: this.extra_body,
             extra_headers: this.extra_headers,
+            internal_dynamic_variant_config: this.internal_dynamic_variant_config,
         })
     }
 }
@@ -139,6 +143,7 @@ fn assert_params_match(client_params: ClientInferenceParams) {
         include_original_response,
         extra_body,
         extra_headers,
+        internal_dynamic_variant_config,
     } = client_params;
     let _ = Params {
         function_name,
@@ -158,6 +163,7 @@ fn assert_params_match(client_params: ClientInferenceParams) {
         include_original_response,
         extra_body,
         extra_headers,
+        internal_dynamic_variant_config,
     };
 }
 

evaluations/src/evaluators/llm_judge/mod.rs

@@ -123,6 +123,7 @@ pub async fn run_llm_judge_evaluator(
         cache_options: get_cache_options(inference_cache),
         extra_body: Default::default(),
         extra_headers: Default::default(),
+        internal_dynamic_variant_config: None,
     };
     let result = clients.tensorzero_client.inference(params).await?;
     let response = match result {

evaluations/src/lib.rs

@@ -416,6 +416,7 @@ async fn infer_datapoint(params: InferDatapointParams<'_>) -> Result<InferenceRe
         internal: true,
         extra_body: Default::default(),
         extra_headers: Default::default(),
+        internal_dynamic_variant_config: None,
     };
     debug!("Making inference request");
     let inference_result = clients.tensorzero_client.inference(params).await?;

internal/tensorzero-node/lib/bindings/TomlRelativePath.ts

@@ -6,4 +6,10 @@
  * all paths (e.g. `system_schema`) as `TomlRelativePath`s, which will
  * track the original `.toml` file in order to perform correct relative path resolution.
  */
-export type TomlRelativePath = { __tensorzero_remapped_path: string };
+export type TomlRelativePath = {
+  __tensorzero_remapped_path: string;
+  /**
+   * This should be set for dynamic variants to indicate what the file contents would have been at this remapped path.
+   */
+  __data: string | null;
+};

internal/tensorzero-node/lib/bindings/UninitializedBestOfNEvaluatorConfig.ts

@@ -0,0 +1,25 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ExtraBodyConfig } from "./ExtraBodyConfig";
+import type { ExtraHeadersConfig } from "./ExtraHeadersConfig";
+import type { JsonMode } from "./JsonMode";
+import type { RetryConfig } from "./RetryConfig";
+import type { TomlRelativePath } from "./TomlRelativePath";
+
+export type UninitializedBestOfNEvaluatorConfig = {
+  weight: number | null;
+  model: string;
+  system_template: TomlRelativePath | null;
+  user_template: TomlRelativePath | null;
+  assistant_template: TomlRelativePath | null;
+  temperature: number | null;
+  top_p: number | null;
+  max_tokens: number | null;
+  presence_penalty: number | null;
+  frequency_penalty: number | null;
+  seed: number | null;
+  stop_sequences: Array<string> | null;
+  json_mode: JsonMode | null;
+  retries: RetryConfig;
+  extra_body: ExtraBodyConfig | null;
+  extra_headers: ExtraHeadersConfig | null;
+};

internal/tensorzero-node/lib/bindings/UninitializedBestOfNSamplingConfig.ts

@@ -0,0 +1,9 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { UninitializedBestOfNEvaluatorConfig } from "./UninitializedBestOfNEvaluatorConfig";
+
+export type UninitializedBestOfNSamplingConfig = {
+  weight: number | null;
+  timeout_s: number;
+  candidates: Array<string>;
+  evaluator: UninitializedBestOfNEvaluatorConfig;
+};

internal/tensorzero-node/lib/bindings/UninitializedChainOfThoughtConfig.ts

@@ -0,0 +1,25 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ExtraBodyConfig } from "./ExtraBodyConfig";
+import type { ExtraHeadersConfig } from "./ExtraHeadersConfig";
+import type { JsonMode } from "./JsonMode";
+import type { RetryConfig } from "./RetryConfig";
+import type { TomlRelativePath } from "./TomlRelativePath";
+
+export type UninitializedChainOfThoughtConfig = {
+  weight: number | null;
+  model: string;
+  system_template: TomlRelativePath | null;
+  user_template: TomlRelativePath | null;
+  assistant_template: TomlRelativePath | null;
+  temperature: number | null;
+  top_p: number | null;
+  max_tokens: number | null;
+  presence_penalty: number | null;
+  frequency_penalty: number | null;
+  seed: number | null;
+  stop_sequences: Array<string> | null;
+  json_mode: JsonMode | null;
+  retries: RetryConfig;
+  extra_body: ExtraBodyConfig | null;
+  extra_headers: ExtraHeadersConfig | null;
+};

internal/tensorzero-node/lib/bindings/UninitializedChatCompletionConfig.ts

@@ -0,0 +1,25 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ExtraBodyConfig } from "./ExtraBodyConfig";
+import type { ExtraHeadersConfig } from "./ExtraHeadersConfig";
+import type { JsonMode } from "./JsonMode";
+import type { RetryConfig } from "./RetryConfig";
+import type { TomlRelativePath } from "./TomlRelativePath";
+
+export type UninitializedChatCompletionConfig = {
+  weight: number | null;
+  model: string;
+  system_template: TomlRelativePath | null;
+  user_template: TomlRelativePath | null;
+  assistant_template: TomlRelativePath | null;
+  temperature: number | null;
+  top_p: number | null;
+  max_tokens: number | null;
+  presence_penalty: number | null;
+  frequency_penalty: number | null;
+  seed: number | null;
+  stop_sequences: Array<string> | null;
+  json_mode: JsonMode | null;
+  retries: RetryConfig;
+  extra_body: ExtraBodyConfig | null;
+  extra_headers: ExtraHeadersConfig | null;
+};

internal/tensorzero-node/lib/bindings/UninitializedDiclConfig.ts

@@ -0,0 +1,25 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ExtraBodyConfig } from "./ExtraBodyConfig";
+import type { ExtraHeadersConfig } from "./ExtraHeadersConfig";
+import type { JsonMode } from "./JsonMode";
+import type { RetryConfig } from "./RetryConfig";
+import type { TomlRelativePath } from "./TomlRelativePath";
+
+export type UninitializedDiclConfig = {
+  weight: number | null;
+  embedding_model: string;
+  k: number;
+  model: string;
+  system_instructions: TomlRelativePath | null;
+  temperature: number | null;
+  top_p: number | null;
+  stop_sequences: Array<string> | null;
+  presence_penalty: number | null;
+  frequency_penalty: number | null;
+  max_tokens: number | null;
+  seed: number | null;
+  json_mode: JsonMode | null;
+  extra_body: ExtraBodyConfig | null;
+  retries: RetryConfig;
+  extra_headers: ExtraHeadersConfig | null;
+};