From 4f8e3e64fc07b9ed7fb8aff6ca60321bc468da4f Mon Sep 17 00:00:00 2001 From: 7hokerz Date: Fri, 20 Feb 2026 19:03:21 +0900 Subject: [PATCH 1/4] feat: generate translation adapter and add translation flow in testapps --- js/plugins/compat-oai/src/audio.ts | 194 +++++++++++++++++- js/plugins/compat-oai/src/index.ts | 4 + js/plugins/compat-oai/src/openai/index.ts | 56 +++++ .../compat-oai/src/openai/translation.ts | 45 ++++ js/testapps/compat-oai/audio-korean.mp3 | Bin 0 -> 27264 bytes js/testapps/compat-oai/src/index.ts | 19 ++ 6 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 js/plugins/compat-oai/src/openai/translation.ts create mode 100644 js/testapps/compat-oai/audio-korean.mp3 diff --git a/js/plugins/compat-oai/src/audio.ts b/js/plugins/compat-oai/src/audio.ts index 7e0e9c52e5..48b1b7502f 100644 --- a/js/plugins/compat-oai/src/audio.ts +++ b/js/plugins/compat-oai/src/audio.ts @@ -28,6 +28,8 @@ import type { SpeechCreateParams, Transcription, TranscriptionCreateParams, + TranslationCreateParams, + TranslationCreateResponse, } from 'openai/resources/audio/index.mjs'; import { PluginOptions } from './index.js'; import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; @@ -40,8 +42,12 @@ export type TranscriptionRequestBuilder = ( req: GenerateRequest, params: TranscriptionCreateParams ) => void; +export type TranslationRequestBuilder = ( + req: GenerateRequest, + params: TranslationCreateParams +) => void; -export const TRANSCRIPTION_MODEL_INFO = { +export const TRANSCRIPTION_MODEL_INFO: ModelInfo = { supports: { media: true, output: ['text', 'json'], @@ -61,6 +67,16 @@ export const SPEECH_MODEL_INFO: ModelInfo = { }, }; +export const TRANSLATION_MODEL_INFO: ModelInfo = { + supports: { + media: true, + output: ['text', 'json'], + multiturn: false, + systemRole: false, + tools: false, + }, +}; + const ChunkingStrategySchema = z.object({ type: z.string(), prefix_padding_ms: z.number().int().optional(), @@ -92,6 +108,14 @@ export const SpeechConfigSchema = z.object({ .optional(), }); +export const TranslationConfigSchema = GenerationCommonConfigSchema.pick({ + temperature: true, +}).extend({ + response_format: z + .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) + .optional(), +}); + /** * Supported media formats for Audio generation */ @@ -420,3 +444,171 @@ export function compatOaiTranscriptionModelRef< namespace, }); } + +function toTranslationRequest( + modelName: string, + request: GenerateRequest, + requestBuilder?: TranslationRequestBuilder +): TranslationCreateParams { + const message = new Message(request.messages[0]); + const media = message.media; + if (!media?.url) { + throw new Error('No media found in the request'); + } + const mediaBuffer = Buffer.from( + media.url.slice(media.url.indexOf(',') + 1), + 'base64' + ); + const mediaFile = new File([mediaBuffer], 'input', { + type: + media.contentType ?? + media.url.slice('data:'.length, media.url.indexOf(';')), + }); + const { + temperature, + version: modelVersion, + maxOutputTokens, + stopSequences, + topK, + topP, + ...restOfConfig + } = request.config ?? {}; + + let options: TranslationCreateParams = { + model: modelVersion ?? modelName, + file: mediaFile, + prompt: message.text, + temperature, + }; + if (requestBuilder) { + requestBuilder(request, options); + } else { + options = { + ...options, + ...restOfConfig, // passthrough rest of the config + }; + } + const outputFormat = request.output?.format as 'json' | 'text' | 'media'; + const customFormat = request.config?.response_format; + if (outputFormat && customFormat) { + if ( + outputFormat === 'json' && + customFormat !== 'json' && + customFormat !== 'verbose_json' + ) { + throw new Error( + `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` + ); + } + } + if (outputFormat === 'media') { + throw new Error(`Output format ${outputFormat} is not supported.`); + } + options.response_format = customFormat || outputFormat || 'text'; + for (const k in options) { + if (options[k] === undefined) { + delete options[k]; + } + } + return options; +} + +function translationToGenerateResponse( + result: TranslationCreateResponse | string +): GenerateResponseData { + return { + message: { + role: 'model', + content: [ + { + text: typeof result === 'string' ? result : result.text, + }, + ], + }, + finishReason: 'stop', + raw: result, + }; +} + +/** + * Method to define a new Genkit Model that is compatible with Open AI + * Translation API. + * + * These models are to be used to translate audio to text. + * + * @param params An object containing parameters for defining the OpenAI + * translation model. + * @param params.ai The Genkit AI instance. + * @param params.name The name of the model. + * @param params.client The OpenAI client instance. + * @param params.modelRef Optional reference to the model's configuration and + * custom options. + * + * @returns the created {@link ModelAction} + */ +export function defineCompatOpenAITranslationModel< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + client: OpenAI; + pluginOptions?: PluginOptions; + modelRef?: ModelReference; + requestBuilder?: TranslationRequestBuilder; +}) { + const { + name, + client: defaultClient, + pluginOptions, + modelRef, + requestBuilder, + } = params; + const modelName = toModelName(name, pluginOptions?.name); + const actionName = `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; + + return model( + { + name: actionName, + ...modelRef?.info, + configSchema: modelRef?.configSchema, + }, + async (request, { abortSignal }) => { + const params = toTranslationRequest(modelName, request, requestBuilder); + const client = maybeCreateRequestScopedOpenAIClient( + pluginOptions, + request, + defaultClient + ); + const result = await client.audio.translations.create(params, { + signal: abortSignal, + }); + return translationToGenerateResponse(result); + } + ); +} + +/** Translation ModelRef helper, with reasonable defaults for + * OpenAI-compatible providers */ +export function compatOaiTranslationModelRef< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + info?: ModelInfo; + configSchema?: CustomOptions; + config?: any; + namespace?: string; +}) { + const { + name, + info = TRANSLATION_MODEL_INFO, + configSchema, + config = undefined, + namespace, + } = params; + return modelRef({ + name, + configSchema: configSchema || (TranslationConfigSchema as any), + info, + config, + namespace, + }); +} diff --git a/js/plugins/compat-oai/src/index.ts b/js/plugins/compat-oai/src/index.ts index faf8e17b01..55e1f3ac8c 100644 --- a/js/plugins/compat-oai/src/index.ts +++ b/js/plugins/compat-oai/src/index.ts @@ -24,12 +24,16 @@ import { toModelName } from './utils.js'; export { SpeechConfigSchema, TranscriptionConfigSchema, + TranslationConfigSchema, compatOaiSpeechModelRef, compatOaiTranscriptionModelRef, + compatOaiTranslationModelRef, defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, + defineCompatOpenAITranslationModel, type SpeechRequestBuilder, type TranscriptionRequestBuilder, + type TranslationRequestBuilder, } from './audio.js'; export { defineCompatOpenAIEmbedder } from './embedder.js'; export { diff --git a/js/plugins/compat-oai/src/openai/index.ts b/js/plugins/compat-oai/src/openai/index.ts index 38d6f80ced..20c0fa2a73 100644 --- a/js/plugins/compat-oai/src/openai/index.ts +++ b/js/plugins/compat-oai/src/openai/index.ts @@ -30,8 +30,10 @@ import OpenAI from 'openai'; import { defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, + defineCompatOpenAITranslationModel, SpeechConfigSchema, TranscriptionConfigSchema, + TranslationConfigSchema, } from '../audio.js'; import { defineCompatOpenAIEmbedder } from '../embedder.js'; import { @@ -55,6 +57,10 @@ import { SUPPORTED_GPT_MODELS, } from './gpt.js'; import { openAITranscriptionModelRef, SUPPORTED_STT_MODELS } from './stt.js'; +import { + openAITranslationModelRef, + SUPPORTED_TRANSLATION_MODELS, +} from './translation.js'; import { openAISpeechModelRef, SUPPORTED_TTS_MODELS } from './tts.js'; export type OpenAIPluginOptions = Omit; @@ -88,6 +94,19 @@ function createResolver(pluginOptions: PluginOptions) { pluginOptions, modelRef, }); + } else if (actionName.includes('translate')) { + const modelRef = openAITranslationModelRef({ name: actionName }); + return defineCompatOpenAITranslationModel({ + name: modelRef.name, + client, + pluginOptions, + modelRef, + requestBuilder: (req, params) => { + if (modelRef.name.endsWith('whisper-1-translate')) { + params.model = 'whisper-1'; + } + }, + }); } else if ( actionName.includes('whisper') || actionName.includes('transcribe') @@ -147,6 +166,15 @@ const listActions = async (client: OpenAI): Promise => { info: modelRef.info, configSchema: modelRef.configSchema, }); + } else if (model.id.includes('translate')) { + const modelRef = + SUPPORTED_TRANSLATION_MODELS[model.id] ?? + openAITranslationModelRef({ name: model.id }); + return modelActionMetadata({ + name: modelRef.name, + info: modelRef.info, + configSchema: modelRef.configSchema, + }); } else if ( model.id.includes('whisper') || model.id.includes('transcribe') @@ -209,6 +237,21 @@ export function openAIPlugin(options?: OpenAIPluginOptions): GenkitPluginV2 { }) ) ); + models.push( + ...Object.values(SUPPORTED_TRANSLATION_MODELS).map((modelRef) => + defineCompatOpenAITranslationModel({ + name: modelRef.name, + client, + pluginOptions, + modelRef, + requestBuilder: (req, params) => { + if (modelRef.name.endsWith('whisper-1-translate')) { + params.model = 'whisper-1'; + } + }, + }) + ) + ); models.push( ...Object.values(SUPPORTED_STT_MODELS).map((modelRef) => defineCompatOpenAITranscriptionModel({ @@ -255,6 +298,13 @@ export type OpenAIPlugin = { | (`${string}-tts` & {}), config?: z.infer ): ModelReference; + model( + name: + | keyof typeof SUPPORTED_TRANSLATION_MODELS + | (`whisper-${string}-translate` & {}) + | (`${string}-translate` & {}), + config?: z.infer + ): ModelReference; model( name: | keyof typeof SUPPORTED_STT_MODELS @@ -292,6 +342,12 @@ const model = ((name: string, config?: any): ModelReference => { config, }); } + if (name.includes('translate')) { + return openAITranslationModelRef({ + name, + config, + }); + } if (name.includes('whisper') || name.includes('transcribe')) { return openAITranscriptionModelRef({ name, diff --git a/js/plugins/compat-oai/src/openai/translation.ts b/js/plugins/compat-oai/src/openai/translation.ts new file mode 100644 index 0000000000..5cc76b2ec2 --- /dev/null +++ b/js/plugins/compat-oai/src/openai/translation.ts @@ -0,0 +1,45 @@ +/** + * Copyright 2024 The Fire Company + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'genkit'; +import { ModelInfo } from 'genkit/model'; +import { compatOaiTranslationModelRef } from '../audio'; + +/** OpenAI translation ModelRef helper, same as the OpenAI-compatible spec. */ +export function openAITranslationModelRef< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + info?: ModelInfo; + configSchema?: CustomOptions; + config?: any; +}) { + return compatOaiTranslationModelRef({ ...params, namespace: 'openai' }); +} + +export const SUPPORTED_TRANSLATION_MODELS = { + /** + * Whisper 1 translation model. + * + * The actual OpenAI model ID is 'whisper-1', but we use 'whisper-1-translate' + * to distinguish it from the 'whisper-1' transcription model. The model ID + * is overridden in index.ts to 'whisper-1' when calling the OpenAI API. + */ + 'whisper-1-translate': openAITranslationModelRef({ + name: 'whisper-1-translate', + }), +}; diff --git a/js/testapps/compat-oai/audio-korean.mp3 b/js/testapps/compat-oai/audio-korean.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..c605c9716f6e91e789316f45f72085bba40f5e50 GIT binary patch literal 27264 zcmZs?cU+Qv`1gG>L_{=1R8%r>Yii&qH7lICD%@May((8`wg7Qkxx-bsHAiaZYJn?L zG&N_LX{KevG8;C1ysqnaKhJ&Nujl-OKY79XJkHPeb9|2D+}h~v1w2^>9FRK~&|fp% z#a=nA-u$#CvQP$;$N>Ot1-`5OSUw+&>`5mr@q?X|8`dJDk>n*s${<=Bf+Wk&E6XrS zHCk3sY!kQJe=jBF6^6KaXiF)zlf@8k&+-=KR3HkTFB9Q|Z6~sE8 zwJqOjiyfJL<`aZmqzP{Vpv-g2?Ox=8DWapsj@mLbP za_0Ef#pQ2XMjMZvly)rTFF*MB>+jZswP*Pw(mkHZ>i6FtaHug9l`k_>x4=*1k_p8c zr{+hVZ(i^@wmWtv!Sl2do8}-g9(8y7YhHZ+vvnGmERVal`{MFf+<(92^aov*BasEw z9=@sxylLdTzyG9NiXA50oTB$ANZp!{OdE*W^(b%|QPGX0?0HTR# z2?n?+LXcfaxyX_{j&7VEooB#=zyo@rX(Nqb`nfa6qj0*VB(N~Y69NX<;}8X&8L}dS zLUKMt=7Q8AJL#hYaQ=5s`@}EO+UpWHKP@obQ8t*I*i9&)H6aWkm5%;ctKKH|O zG1796F_DOe2rHNVf;U5LnfL-m+tk)=pLf;EOU}we>A044BOO?hzh#QVr3>v7KOcOV zKb$|&znT-clrd1OvO~G+(m)eG+cd*kp#uq_lQ}q1ql64@X9`ak5&UbP2!(*3k&^L6 zgsK7n@GpP23_~jD*$jXJ7)%SLk-NRUJ#g4%%TXkFR790(1}0MlR8)~9-Y2UF3cly% zjt17`tjJXBtGp!1OFL0L8QiK;;qr-!GG$%z@gV$)0*0|KF(DAA`^_)9Loc0kt9ITT zl4Cne(YO&-C$H@QSJ8IJ#2aXLxYpDt`+>;hC&F?lVeEo&LOK@TnP1J$)HjGroEb}> zu;m}+Q*=kVxo`$J$yE{16eM7YcpA6Gp26-+6~pVFbDTjI1}LuV#cI0ZvD*H|_P1bA zIW=7|aXAc;`>G`L{nvS>PYwTqU`0MT^~`FHEx*4B@$``@>=Hnz5YZ^Z>&)$(Yz|90xv&F1=$RXwZgiz9!}^&s~r zDyaueEW3s zOHS-B(%HOs%J{0Xf~#0Ib{qeH{5_2f{ehMFIS)lVDA^#-3KStkAPx&+=80Hc3p?BO z%t(ey_W<}E+u1jWcqZ{u{~Az;w5NhX(vuf7S)SVBKw8TSteRLYtAa?(7)t=*{YsOp zWvCFLGFKf41!5yANPgpeIGz8dcVbwCRYGUpC?RuZq*<;Rya6yaBU}TaYwJ%lfniKg zL_bA;t}-+}co9GDU}=n!m5!j14Ome28*GhTGBjx?JGk^xk`R&!vvq~`lF15S6J{(Ogx|O%$6}&2(z~uG3BM{U zXi>bHuzTpB{on5OPRBbmH~Yp<*TTkc+w$4Wf9u~<*N_UV&CijgiAWmtbA+8M6(xh) zUD&U$rXv722nxS)ArggkG;V6ANJ3dK%>ou5eUIp9OR6`52$Ykp(xf=$d(a|CSq5KK zbN&>llxZ;>gHjG)g`fd3B7Orv5$*hT$^285(z;1cdMsrkN#M`B#r!_i*kxW3V48vO zc1uEnW)r96B;-T`f_2@pSJ>oCs8Kx}fu?mcn0-?q;p&|FG@Hppr zEfnZ&irFb`U*$^=CU!{akvM9g{+A(Q^5X{SH7n}FphLTo+(Jz~hu-|%Z8avlvv;#I z8&X1Pi#DNJsh2m`NbkB@Wt#ED-@atEuNN|6P6C_& z;p`*INt$BK5{XYxreKh<_GjEJeA6D{KA|NT$A=WSqX$7WBcL+i(*jy@S!J)SdNjXdtDmS(XFRDd62XiEWdyBL{eJn-VWJ9(0 zv5*eqsFe6XYlP_^p@_qFOQpPgH3rhp)CUk21>A!=30}di z+V_hCKBtjV-+VRN@8d9dBTLo(B}eUGF`c7>6bOZc;6m`Ja7#PZYKe#L%`o>V4cSJw zQ-#m^GSCe(U^oD(3PTeMbwT+)PoNSj`>i$~Rb>5ivO#F4^gwDRS z)hG{w3bj)zLe_dR+D{B&BcA>A5iTk{G6_a$(CTk8HEE?O`8XrZ40X_PV^q_U>(vXL zszd1pkp)KAf@r4>C+ldRqORCwpm@hJb9ufh70X#>H{EH5o{o=4CylpjzcB58>3@Zx zBJgc~4gt8)@hVti3Sr2*3)=))WqxwOX4Xaasj}H<$QYB3VON#^NPeTtHiAmiDgM|{ zz>dz*WMde1^i5GUfQZFPu|APi60SJ&t~?aNk&H7H@MxSstR<+>lmLUVGYPITRJNE{ z3K$z1Qiv+-(8ua`n}En~1dC4`5%H@~V*>_35pJ1C^14s^sQ6)Rdz^ zY~78hddSyeC-a0U@Y1qNk#Nu6BJyNHe0Nr0j+J9CX2L;x zNgVS3>c9Irbt3RB;<*e20&A)wI%37uHBxnqB{#r8A#qj2FDjyqI3!A0aFHQAQK5V! zS^*#mnHUYe1*Re`M)3kzC@70g6hhN!4Aor-1j?OQ;B~z!A=kOmz2`;MtI6CCwl@A| z;7!|4+Eq2Ha`Hr;|4w>q-K@t`nGj>8FTN&B@L~Dz`WUf#P@F^<(Z`B2;)N%cc3%c^ zILl*%?(9+SsOfi=pf&Y`mhUB8TE=+c(uGQ2a}XdWHX72t%ff%a>E}qW_@3vrs|Px3 zeH8UL?eY^w<;aGR`(xM(lLvGSVz1p+I%sdD*oKJp!CpAxd%bArYts6I_z#+0lUm=d zpY7bg`+@VG*yX}DCHBPJty5~sJCjXB3)2sC;J&okO8RP9!`=(oK{CSm>rp#XZ6H3Xcj6-&u7f6vPNZdv-F$=G$l27kvcc0Be%=&pu(+Vzk#&W#I1!SA6bIyL&2 zvYw6$Mg`uxOBnd4_yawCsnq}UKYT9*SZkUiK#j?z2^@n0LBi^*O)`@vEgIM-8dD-fE*AsmR)T>_hAE}XxkNlkjMd91V+-m=Pbw4~qM)#hJ<5M4GD z;A*b)8%uz)AdW{$%QB+zFh#8D2?9<4kjGQ$cp-4p8%#=KhgL8(8X!Lw#Ym=yvUxHX z(U6(KaRnQsssnQ18BwP@FMpR!EY7blQAEZpDULcIS?oq8na2f5$-M0DM?Q)Ko5svc zA>G3CAB(&44xiv>=RLdd@!`wgBm0mRKAlFu_|C($l#s3D500@K<582!(O-|hmMyG3 z;U{|H+I$tU;#E|`fv0C2eIF8EXjkOhJNB$7q4~6~gAU{7yVqos-$ggs_|%V2--8~h zeuypd@cQ)mP56y9vq5o@t@5)H_iDAFWG$}h8@UhDSIC#fHZz=?Z7vGCC#f7fv8zYu z+{EzQ$^MXH0W0Ms_p_TNNJjp_%g@@Drf=q#V~@NT|Bt^1OF4X-|MRGcdOScZ_SCPB zM2qwSLM$_@D^}z&j2R58j+Ks=qcg`iN6s-J3zWF;ZQ7I+9!JdiKtO{wVf3J^tNgn< zx%?DYvZ^Ays;m^5*(c<<+^JjLb0yCLo-1Q!5j>Uy#1+ZJ%DZzFY^hBTl@;|}txh6S zoaPwX6lFWmb%fs#yHq)scmHlcF}Aj%laZ2Pr-Q_n7I0L}Mu{mI;{ioP<&^O*QZKmD zR!9d%akMgR`Ce(KLqnxi^J&NT_iP5Bq5`A)j;cxeQ3jh!jSise+rAL zuIGK+|Mh{#{p z`k0Nv$=_N|;77Q_SqVK?9Z5feC{cZ;aiIGr2AyxQh273fF8$eK7ngHKr02bH$03o( zGQCi%fWojRdq4EFJe68A-n;nheVZ^l_683v?GPq3Uu{y+Zz zm3GXw|9l?#?8Y8+4u9<@1nfhkV7TkzLQDa=MtGl~!*xxNd6KdroYWcqko@NDXCfRw zl8gXwO-*oBw`21$eK3%*h;g*0A5j@3TQAK_lqEH%31O)W*Ri>B_QHbV!n3CtW)u7j z7PMa(3Bt%9Lh2JEOa(EbhNJ}^RU9rzA~+C`Bx5=!I66dUEL&Iz#iqNZfvyN65m-hD;pr?}~`-***mfpSG?6n*VK?vub zM9fslt?5N7NJdnK4NW+h_Fz_g@(?F&RXqqK9c7o`8R6vNp$hbE-S=%x;!fJ9aHrIF zynK03=n>iH*yos=M=kfu$Vvnq%zX>D!8q$^y2~a!lDmdBXu&^o!{JKIu1Ff1{ci`r zfBpw!7!JIan>$P0`$`^BERHrh6eW?-JGOzkpzD!b zP&&U71byvJ<<(w)D9>$Y#UTCcX>1l{kimAWiEgz@=e@*_l|M~NAh1;}X-!S^qkc%h?#C*>_i*Od5Q;-mm0wQ?zrs|IqjOdmqk^uh*SD z(aC^``1Rj=UBGJ;-NQb))O*(D$5~N1<)cj>E?;(O!!-HzodpHtJzpGm*0aXH@w)cd z7G~z!XC=(c-xG+j#>a*ZKWTSir1HW!`qlzc3!fEHE%mQw#l@|3BgQ@mc@PsZ;t zy`A}5;wT3@^8fgs?fiiXd?e1fPrNOIie#Ls$Z!Dh#Ub|L7Ab|7Rj`V1C`f>jIorc` z(?S6mGiDkEBS)d2V?$`7$@@WaKeucN$gx5TW*c*(g|ls^Fb2lq7+E4dL%%z~*SN&Q zO4s{6^3#!H0&O^uu(&bVB6GL|ZI?3FjzpG*z9Os0z?&u_6vZJq!w2YTP`)Hdo9GmO31o}Hkm8UyXz%I0xKl5tnkTKTTeI%O_vd_d z9`X4-?|h>_qOQ?vGq20bcEsn-rI24z`)u-cXIj*B3*YM~rrAwKC`IgaIaP1=b|OHl zs81=F-{4>K_LbGC5I5uzdlgw`0-TMwlbb@Kq&u=9#?N`h}f(D|mKvTlz){CwQ{^w02Rbrk+P$~m`%}m?0 zSjVo5%GJW8=NnCM*oiFrHN>m+6NUvZ=jBcoz4-0>vwz*^&Kae;hMqag$Q#A_(n{>u zLcw@qlnxF2#bXFU8t_SWGt{+;=u|lR=F;1p1~O*L@6CRKN=GoXonh%YnLfH*#|JNH z?bTV+?^quld7oCwMLNH`d&F-0fK zIJInuh(`zW900I<4FV}9LfL^-0i5n)(bzevv@*JGc~oG>aNs@}i3)k`v=$Q~;l&Fm z)FWzw4NQ?H_M_IPimeVLR8|owcwEhK7ez(xO9I~S=?5$jOhYM9<0zayGG}2B$g|~^ z5>Z{}@51P00qIv-!T?`V6NISD=23)L#vg`%okyV_ILleA(Lt65Lc*|&b4O($H!(}z z3_3I;kcr1*skY#&`hz}vj=V!imE?U{@&o>F&~!UnbQ+TI`&d(Mcj^_yt4GiB4y`G z3#|~9L`qJ^=b6fo)o*^}0cTIH8<^JGxUE%KmXYO|H*$8wf(pG&xH+3v-6-mq+Q}dbn|PB zK(!lzhmzoBCIakiaxqgLiw5oa6l@U~BI=uB^jmKOayWWV8hSNYz_<#bCj#2Z?FjF$ zjy@u8o`B~PlG6?=TF7j=<3J><-06BT&W#8)u?2%nKtm{-jXCP~$w4u{0l=qek8_AM zNC0qFu_U8ND9irEK=4;KMvjK}4br8PQTqDc>W(URAsS{DJ^xYc(l$vTsu;*>1I>5w9J$a=z>HwIGh0adIm&XF^@pKQgV$J z87DLBuiHFa9V{^s>dh)Mb*XhMk0@3tuDNNW*Io#>Q2D`X?M3}t|L#b`;D7R;usy^k zuy1W)lpBY#UksFl6tzf~Tv!mxWW(9(bsJzC6Kzu!StAC)ML5=+l5Xf~8DBSRH?8TOKi z&{RWrthJ1kQ8&X`U5(Q{m4wQahwj{|<9kaE*fT(GQgwnjeDI?)9jByFA!Dv$r4JJM)VSserDRuYN`+^fu;-pxTE0BG&*fYD<1Y=b?|&`wqm&q4so$Tx;PQK=Z9)oX zksiCVm*y54)_ptALmi!KY{c$tQIHS}EyX>JRJa0FI#{YJEFFH}lJ@3I>Aqvkj5~s2 zCTHZ19lfeRFTr1viM;epDLy4jH8;!d*aXVO4Eyff0{=mPUsSxJVb%Yafd1!yY`6W- z3URLWiad&URf$FhiZ)4Br(7@{LRfBgR>^eZns}eG{9)eJqNCNuJ2TB! z6q{mYy65|IHt`iqyY+!X1(=d=_uaN>xoP4t45gpfT@5;Zo0j__D?(fTC2`yYXsmfc z=37CH8``16VGU~a+Oy$@L^>TW9rpk;!$(6Oc&)pv9!}q^R4)eWT@09yR2rnPQp&pZ z9!5S&{O2an|I;6j3d0DXM|+M@>%i{jbhy-gKyr0sPwvrLiYjD|_&C05F{USuDaG_R z-M!$O+necvet6|J&D(9)L#ZA7<4j`{h8Ui}05k1D@@{APt?%?eD5Z8KF}YA&8y1gZ9iW(^1hYtVvz89px&0sE zrN+>kv4siS4pKp=y|n%AWw& z3Q};C183p0!4!5a1S$sg0{I~Tq6m!14DvG0l&6oztmubvDY-B~OKamyGB9~x9nYhi z_x2;MRq-Nhw77Pe@Mxx41+iwdL5C}UHH0)2rGV8S3hb5hg)*U65m!!tg2|`A>iNhB3fd?YXE+0x3EB`p?bRN4m5dI^BQJ!wS_flGi}* zI^}$Q`JM9*58*zc-t!0banq>*vFIx&in8YG?sJ^oJbe+nO=3Z#nM%3ydvbd!gI1H7 zk*Vvf)G*MU3X`}u7{cw?*5xM;MpJhSp4r*=4EnHaWXRPi_J9h&WGv7Hm}mMx1bN6& zn9*bL(S}^4Ltq+04FmDe7M26R?8pEKjGAaYsJk|tVm{3@g_RV4`{%5g%F{Nx)vj=0+>}=qa{kBxG z0Dzmkpcn4BGjK(o0?C&YI*1sdW8(<~B>GAj@yYNlF)Z9MrqjO1@Kr@FK5_TGYKHVt zF;y2jRo+G~0jZcn@ohp&XOxxM{a5|}bN+gK+18(V;#}cr!V><3{(z~cwxp1#FsTA^ z(nBXZTE$d%JiBSiwQ2dfL!8Jrs<04AInOceko_PwUMNV!H#-(&8GHKpwep#mS8Hd$ z5UwR7EQ2c6&_qGsJY-c`ezT%eE!7RTiEkX9 zFMjKuVhhi=&gXNBxz?6Bpymm)sq?P!Wh;h|N!F zvuR1k5Bn!F-79^y6Hin#Z05Wi^->Ov44K(#KtVn$ez#0!3Adss{`Q{yV6cGi%V-T9 zL;92az2g!SPnRQUH6lm6)la2^&2Y+&SNXG7HA38ts*iNRTaS|jbpNe?Pvn2|FMpd+ z?m`bD;F4kyiH_}FG=<7u{u`nbjN+AUke6(qegP>Z0|L)TO;}3-ED>&o^9{RIlcWG9 zN8hui;tBNKz3OOD1OmvPFp+c>M?s+Cx*JN;^7^C*1vLe|h@1nWI!iiHCk*;c4fg4h zA?muifQg@Wi;fNnv>Tf#dsJuq$qo(DVl&pRSw`Biw2+y9=KhkW8hg)6t+>xk_o}WJ; zmM#`A(NmLKm+Y>D4!GJ1A$;+@ReKfTtZMhHp!|e}x|?m;CE4#|w>50qxOatSsC^Bb zgNfE_616g|Z5tmtp4EkQ7EN5iPi9%(2@6juJ`XD#(hXF0Y-#$!yy=zaxB1DOEsv{| za<)guNQI1uubZEW^2+NMw+U@-7HJW&EeW$DX}M%qcj3EqiS4^(anGhBakH~8mGSg{ z`McF`^WXMA+1lCwIdX**EJ;KPB48HI`;(m0fCWXU1bg)W+YFNgkw>}mc#d&u-3TI8 zQVv{V!PW4S4g-<5M^YaA-)J;XvE~A8aYYf_%mC39dUjzn#S&52Pil$D@{#O{0vTRf zaw6}DdHr$ynH>{Ulw2rF8I7kYDvMf%9A44Y4!TKT_j8lrNSwMhRY4ZBCxTn*N=izV z+HSavFvWF>XfOjheS}D1zkxU1umsXkYhc1fTYXdwjDR$jSK;8*fjf9z8aQ^w^~$N} zUa`Mb$JEhPsdlJ+X1yb(3ATQcpHYdDN=P%m6NM2;T5fp}yRY2b9yod~U6lOxB7c^j ze=joQ_Y04Fxr-IqV|m|#6V<$+l*vnRs-_l;xu7QNCK3KsS!D&?95Kb5v)EPs^}GsP$x)%P#FyN8@n77o08%R^bDD?QW<=x2GWe@ z)ZHOxuk!F|NW?xtKJ3SUBlwLGWNaKolxBTNfY|B^k-Q_TD{KkY(>EqH1k_8f8~kef zd$7GB?AMQjqz4(iAKxDSc+vOOtLM}0d%WbGzn%Ny-|Fo@>W0NzHJtCQfDMi%=H;83 zbv5r|b$kxf?0qw3f^hT%2mWlMTp#i{+#5o*xqEm=o=y$z)n5ZMiBif%P{C0i@oK(- zb4}Xag6O(slm8=s(D;Y{r#WvmOLaUugvQpAVFTN#P*T=-| zKww|qPj0eJbu#LVyVJVI^gGN0RdU1~s!i1GW2we>UEsVnncC|s*|=sP>gT6<;qxEA z7t`NePHjEj`h94}is$RU?VB6x^ZP#>`TOph&k4QGPKVnHSN&{M%RB3G+e4S!|9a)w zFF8Bt`GTpsxi)=HoFD2RYUSOI$M3N_Owg-+#xfxesJw_jRVsCLWwr0bUcDbXcqaP=dzf2$lkD$A@jLXIan$J4`rDh+>Qb5suWt=+OW!kON8=kpSzE z;oW{ZT33%EY6&oGaF=$=uQDEY@u_NMMKSDo_?o=`p|9UJK;|L!_-%>2mm*eAP+(Wk*{U%p&cJEj#A z7_4>p?c>3gsjpiIXe72lPw?6HaBI3B*Y~XkN;GfZ1A|M{4bcw1je?+^+_IQ(8P@id1nGGm+ zd}Aj`B)fq~H`8b}v~iuJXLkG^8Mq~|BLNahG1;Mww~T;s$vO|MQoP1TzCEn5ZV6nvtAx8 z`3zN@Q;NPC8b~G1GRc(aL!Kc;vbjwKdO_f{eA#2MyUFjhrkr-P=79^HOyy4myhPkZ!~texxp zv0QS%mbkX-+HKY5h9G`q_oPqfvHhVpqneYC4{nr>6bX+ToNvE;D6+}BvTM5K{G#yW z>?sdny9iC;a1-+6qS98@q7QLv8Q8q*fKUvtNLdh_oGj4yZ=A-|9|E$y8rO^ zo}(WIDF7DHJMWBkZ2v1y>fm5FIE6|)O8t};RR1?};Vm^T#*$RZ>5c*F!+)FdTaYK_9G7QOb`AWOBtA zc==Og5S38!=!G$bg%s_ufP$g)6s(wgH!8gA(V7In%p%fAS%^n~YT*pO2zT`F+#$J=rSoCUKtSQXlPky#|c>AN*ysPuosdHO@PyemG zvT^*K({Oa~Se=~ZkBZ>z)gt%&;M@ki8*k^?Jr87r6|q`-l=6&qm6mtnF5Bp7rO!Wh zIx{u!sc2FpK<~(#56bTy=nZ?N+u~|vRkMS_f(}i+|6#$My4bO8I`uI8j$STz`}F-E z|L^w2@Sps1F1Cirvuh}bBv&awGgMWMcFXnCa}Ye!RJIBD8nGNfRlp zoy;KM&A0>(j}(K1D#E4NHDN8y=S?baUF*s4&#~P=-#j{!u02!l$ijUhS=xd@q!9C6 zs6qiKyI%RBOB@G_A+UW25!wdSv_+BJ({P#XB6n!T9V z;r%9Ap8i^rcGgJ_i|)O5q@cd&a>>N~(%DyiOw$L80;UE}X*UixB0o;1y*G2_7Nq`f zx8OhhssGRaoX$~1qhxyavQbbh99$6CSX85tVy2)mo%3wmRKYkw5UOM9reh;OvBXg+ z6hF>ol%N=(49@7sFyO7UK8qGx4PE3ez1|%BPJ6ug#x6_kQ7RBzSvt;==_Fj^6;?1V z!P%C^9Y)xN(#!I)Xaxv~pa92`7V-vYCQSS-K$w{2a8Mhhl#s}Wt3G}@Owi~ywRL1! zDh0^X;Tft3w{_0Kf@AX$@OL$(*aRUQ5t<0LJsXUy#DUDc%Wff&qP z>mT3GWL7#DFmuc$YRdfg$YzeeDNs?&k~~k~-a24xZLcyXE57a+Y?J-!dthwpxd(Bd zPrA?RG~TP^Ul zyY#)eJmVL{OgCiy8XAM`nZmM)Z*{wJ+j9bc45nOx8)Hy zIT(&A5iW=aGQ{Dd*t8W9#`aoH8GTzmu$xg+&VKSBi+-{qR#8Ji&~^#83$96nIXcsN zlqm-TmSDC~&h-dSeP>fUD^htva*?i}!^r?>8(0|mhI^w$T{Jk+JYnOIk=!@ZMCsZn zlB|j!v9iT*?&YG=>ErIJre%HA~Q)-uH47k6hn=zg#eR#aDg# z2BV9>&e0Cgoa$D5l-6D5RALFu&dlbTT#BuE_|o8ft=Im^9?Wv@pD5ebQXxXO-BtUE z*Y^23tLjD?`A!X`)o)2LKZL2QfBCzZ{lov-+&Kr!?(gA?L6>PV?2*CufgP1WFj-t|uV`BPHQLH4OGNwwWHgR%bdeZ&m9PnXDRejwU zswWxMoPs0RGRc^zvhg7(K?e-f+Y)K}l6+Pl#hnN)s_2IsG=W2p7LF-fb&BhBVC)lu z&zJ$IiY&cbaz+4N1RDHCbp}hsHoGRzpbgX=+$6Z*ba#J&f^3d_RotU$le~T-y8e;` z5j>%8MwE!LO}p$H^RTmZ6(5yP`gT$(i{5=fUi#_Eru(JyE)Eut}Y(`8nBqfVi(;8Bt~Yu-i2dhWxIpsB}msu62>8KH8)Go zU^3a(QR7Q!eDy!;4mw&SBJp3{|6d>f@^_us*8gq(#-&)M{{`zHhA54wA=Hel6F}~i zLV7t{CU3~IA;nb@mW96Ss?;}*LV~8LEi_&3r`}!^79$J-<8`n0I>y!DgLRNQ9r*^g z5P$`_Us6$0!7%9YB|%+NmIj~i@UWFf)UO$SyyTSex$(d)q}|wV*vx=usk_bzj;4Qi zv0(^sXSiOR4PC6jx(3*an2Y5T34je|l4i5UbkxEkB@s9wAvy})j1!R1DtsoB#=r!V ze0!4<%`0CPUM+)JXA`uuG6$!3Yt>pwC!31tD=*C?bRRn$Xjbdj?Pjl7{JAE!^4Q^o z4ng}z*;a^suW{>@<(s5-&!1ac{pV|IZr{BB{cm&1?5|1Rt()r~mdvWF*k$&@x#72Z zqavK|-VJFM6^>!koODGopN)=;Ztrv&Bym?fq~d~{RTlM&*AQP7vNor8p1J3S`LZYS z@)J4GDdk%`lV^RkCqw~1@hVY@wX`uF;mzq{GC|Jm+;?Ep&t3DAvT zS1ps*nR?wSikZ zgvfS;G^<29U>lC>#qimqGVZ=2P#c0tv^G*%fEa0tKza03+~_YiGx-p*!+RxL7k+1E z6p+y1F_I(^*L-_nP`INYsfksEqPN^gzn872^mM7AX6H(h)lWSJ>!m^O@1v=TG=8;a z;s>*rSIp$zvHKYAz1zj>Mk{fr9}``=6vZ=)EU5gAjpd&|@!*-;X5+1Q85WHS@k=gM zbzN3RhVO3u4SCSsl_=f5qxAk<^C@ASmzL4J!?kVdr4O^u+eAaJ)e7CfCOa0^3gys(p za@k_yPX-fgUtDw*0s%urdNz(((6}1#Sef7Q*f*L*d-99w1=ujrBDgZU43;Ju7?1%2 z103!vToDGuW?u-1P-C&Gav%))wsnyWzwbMW?E@Yz-NLnqe8$M-(Tz)mM^T>LOmRhl zEDRU4frg|TNy(UbMHitd?)FI#9(WHo-ZM8%!@f> zixI4X!=*tW1{Ys=0Bo9C=I$q)C;;>$gtFjfcySQKhQi7od?>kFl(8$w-AYg0@p@Xy zfXSlKg^E1O&`JqiCuRgJ8>;-j-G{SYolZ&tH7;b93v^R(g=b+oJ5-txrg|C-JeiiE_3H`co=ytH*4MgBfv1E~C!G4V$oYGca6O~I-j7x>3 z2I@jz8Ly`Rq-2ht1~Fy7q_8c3Fs9p|z09zH(-YT-x~L;g&PN0GT9-q)%NKfmoc00? zGSyJX@z$k2viEH5!57otlr*2Hl+b?;JvY>Le_SGU29#*tS9P2YA$-; zqE%n1)l3|T-l%)~-J$)p^04mr+fq-TeSEgtCH9VvRmCg56t$YYk|Vt{RV_u0 z^I!dk8(RL$UlEN+33UBaZCx3N5rTE5A4a1sY*H*54}ry1xs-~T$NE3a@q-ljVqMds zAEi4>V<6kJm;9>7j{S)K1`PrU-cmH(05f_(f(a^u0Gv*JgYjJ$pb;ryF-GBInSL`C zq4u7>O`(d4*qXgqJ_sYS0zj(d=w+pDsq(dS4krRb8(5*GAn`CW#&%9&3N?i&!W95O zT)>#fan;zHmJT~8Kqj2ya`dSxNMr^~6x2)qB-xu+audl2fqjr#yQ6!oEwuQN>(4#D zA63%PmWlCw1w-x$LGrgA|9SI8##JFb!&Ynfrn$qlm^^u$myZW@_x1E?9f5_e>EofZ z#Uac0>%aU6d1{*7dMkf!`uNd54N46a&*IOvoNe&Fa?&~AxYoHe(Um&L&9-ODha;lT znI39S^~GMkxJwj7j!;Owz$!}yD)2$*A_I>vIFcol zME1ZYGI-VO7wlP(0i#Bc9iBiO5X3_1ZkIL>vUku43}hhR8cR|3S0T)Tm1yD)Z;xzB zLm4~%4jDGh^q2a{-pN48ff>LM)qb?4yOxV$W2JS-?=vOKl~F44Zgfyps7Zx9R0Moa zBNKBpymXAkfbEn;gZPqQa7s$Csj&i(0s(2pIU+?5HAV7LGC%lW0Yy=iY%H;J$GB9*yX-76$~_U--3cEf(=Tve?|kJpd1c2#>}BkCYGL-f z3+&r^yF8{J{(bQK{CU6aPd_`4sPCw}T|KFg~5@S$FP zN@aE9OIPiaD%sbS%fb;-PPbovYxe!6y|@Sc!S;BXa?Ktrt=z+XHgrRmA!7v3D!knx z_T>NX{Pk4a)}QaLb7DX(U0)6xgcK_e-hl#Ulk80j2#K|fjzjG1tE-6v;#f-|yOJ>{ zb}4-^fl4fVEuoeaXTpXhgfQM@CTy?ZVRzaZfk%0cU_K0q>~-ZI1KCnsE;YN_S|gE9 zM@H0DkT4jZtUnwsNwJ`kF*J%{5+fobZj5d#B*f@e?*}K*sUIvrc=D)gYCklUywCvU z@d#<0M-j+<(2NVh*-hP9D1G>uWYfgEMM+pDoI(aA_9%P81^8N^?DsM#Y!?EYkSIZ% zQw2dqM!|3lPzIoQqc9)QWbjurvPj4PshSsVv8zGMNG`Q*C>RMWd|h9(vfM9u*FOu6 zR&F3UAZ`159Vz!{lxx1@PcMe^zm1$ad%CvsNO3_yLHp5T7k!kU=xl8q(AlrE|H%Oz z-M?$UoZ%&}ZC$j`^>L?IV^c(duPdQ-)fb zR`P^1(t!yr(f{xM$vHz^;Jf#>{zE91W>`FFtp3Md?1{8hb~hT^pJ|)`-zrcSnN8l_ zEmACF+;I1n&PWkUj5s2V$fjNHxQK;=R+(%&?Diy^(F9^65{RFiB#=uUv*orn3Wr~m z7cBPR=?hkdL6P&$I>^L8IJh4~7obymA_m&4ppa#tObk!wj)DwXcoLJtr2cxFS++gr ziNMPM*u9cF1gIRN9$s-DHBk)!Wb}J9#^g&-fE*T4xq6&7u;2-Zf{TRc+&5(R0X|xg zj|LlWZ*Cj2vBuuEh4tD@XZ0qfN2gq)(dCurYo3euHMbZXkP@#bF45XMp*nNLwW<7m zvfIvS@0mKxUN@BqO^>~eTHhw`AN;NP%&qalSEZ8;eulrco^(&H+^%zX@0wb_yQF{f z>)+W#gwES(`?s#rx4uhuwEhS8-7PU?EphDx zmuwjve=eH9fAlh@?f=^UKW7LB`OE-B3A!>-ld+fd^NBY}yyqp6$}%J-UXmt$WcCq{ z@H~iU`XPW0rtW7_sLqT~syYxs4P~fOEg0TZ5kQ`*L5EYt0W4J>u%}`G4XQr9f{LJH zEWrzz_x z#+1(j(&!c?B)npr)hpekhY7S4CAPkO;&B%ZdmZiK=Kht`zMs`+LasYccJ$mFVsQ&5 zkXtP(V3}soHKFIiuNL;)$X!pVkM}tpc~>p^iWT9}xyeB{e;wal_q@FZzLnd*hU8pT zf7eR#SnoKIAljvW#Ig3GlS*Y9Ue)%whQZU$jrC@b*W|%^SN3gVn;`YDuG{;>tMdlc zR9}aVJ{|PgKWJ?m5?S33&vS2w-aFsm9F%+Sj*$UsBzcu%smQSwqtjoTZ%(hKOVaw1b{wU1fCwh4 ztinVf(N4y`ze4Jw>)*RWshWloY>p-8GtoFx83o6NmS(f@VY|<*ZEgJCnVg9c@p@d= z1)A=fa8>|=$bAV&=1B>Ls4?T>_v_nlj&5Y%S!VA6f5}iY3577a_)T}H3pIt;Pw3<0 zr6GArvq2ef{~YF>#*3$4tZnzem#&dHYX`tRWnEkO`%gYwyAs;pr6R2OTIbcE+4AvY zXjES_+3GBdnIvHMTEO^J@>0^4&#%`Xzi$278m3{yCNxXK#qEy{NS4vM){}hP``; z%J7vzk@iT26?0RUEk*|xKwwZroDNc$hyQi)m>I4g&K6d)Um%NL@E*ncrYO9%sx1ay z-AXJMEYHXY_Zu|&b(t}SOl3)A(51#)_G@%@K0r&tuiiX!(Z@30lqA{|_rx}$ zK;B7fvq5qP=BO9YW3L}8Se9I=JsoJ^EuA`8zFzp=0{^7^^6#Vz#umSqE{j#GUwiD5 z?R)3)lZ-bVjq-1=dDY!!z73U2DtTvL+j%^s_Ie`{osY#67Sz9UHj&fJ*k$o4a%G5>p&}qd10>;oZA3 z%Emjiv)-4LUVXIuNuv70T5_4h^V#r#za3LvQ%m5fu*c_)LQk>4q=%bv5R8s zA0b_k)kA-Y3dyx9jX*3D*{xh^rK>8GNR(KxVW~NkgF#r?LmFLNH986@00QWKWF09` zKLn;Mfz_8}u|5fcYzEvC`)9_VvvidmyK;BB)FHg_Alehgq_f1pBqw%d;{m$bm(siV zR@FEAmhdLrD#({@3zy5b$ zjR)U#;=;^L=jMQCQ>QaM6s&(L?Y?JqMP>ec6(erUTUGYUbfup%;Wg&0^zhqNa!Ei~ z2?HoG8-;jh7E;zIM+ z&x1C;4}VWzP+Qt=#s1rW@W?RyXa9UovzFlsWt*T=47cM=Ca%AJi+_p9;?`g@(2wCf ztdK3T=Nwq5LlH#mSu5TGm(zybzr z!jJ{*AOMaV>2u2`7L`GSvn$b|?a=O(LWKiBpa_Wm6010t;5puOa)E3$Hy#7r#E_=3 zyiD||u_mHSG(umAQBr3Msm_Li2D#twh%*5)LHM&Zh`6x?iWI#t)><%T%7FnBbJ3Wo zw0^b<{Z7wDJS2bbD8*PK)8x>(`}QI_>RE-ByFxwSLo2ipa>2AbQRuxye9N|)ako!g zxgMwslm?O3&f}=8{(_IpIiwgVC$~pDf7#vluYWzR8TkMldaoD%y{gctSjRgky7#VF zb~-75GgX&QWGgUUn4Q5M{OdefdaO+VtD!9Fn+3!Q@!HsX^2vr_VjO6!xf` z$N0f53x#pM_!UY*<*89}!iEg*rjIN4c!buRM8)FF+tHQLt1MXscJvv8P(FJ#>o&ZGsAWYm!X|z;WkZXNe0A$ zjC6!z5VfLih4gM4Os;I_IaxImr}@to7>*S)Ax>C!_DB{5XKnON! z7wX|YvWCHB2%<88u@KcMgxza1D3BFug+?$j60Ei^3%O$ju^cK?3&SNI(9Kt2%k*a9 zN`x()g&MtUeaUil4j&Oyi{IE?vv#qcQ{(+|`3XtV_9>qdSlRC~*y$$)b!Ttw@O@a0 zGgk`Zm-Lu>=cxV6__d>G>z3KSSoN=9gK3e$DPNqopxyWhxxYfb%RVpf?OUs_kx&x5 zP7@c~Uuc|?z!arc1Zi)2W5#SjKgyBo@iupqVR_)`TRVRZg>)PFudUrZxs$}-ite#! zuWZ2$g)Hdvjzb%z$J)$Wp$QjJ#xbc5_nFE{t80}D^ zOlF2eVaC2YPk-=q50}i*HpCcjxG_7Bse;7Oi^af2!RQ1F&R;VIII_2oHgw8mlYl@N zU8bYK+i46A3~rc{E^`CSDif>&^0-r}^;Rc(mW8`j4(Eq80rP4mySD~7ef%Q3uX zcG_T5TFlfjR!>{M%msy=kRR-1pliX8Ra^Z5YJ7x0w(;GjS zPv~v_pW*sHjsO4dKcCIN*a5$f*3bZIDOzZd0V|$2fL;Z)qU9&?ikG#0&?8!t=w46~ z+J^jC6j(6aLJOu&f>Nn2uofRi8UX67BnqMls->ytA##zCi~vEZ9DtMHi5so$SRs9%QdZsT41 zAd+dUA|QU)&-x~^KhchNVbp#}c9KGpiuq*R9s00T(kW_;oJ&loER~|~mErb}nb=!4 zKIsZ7sq8Pg^!}p5uiC{5*;c=zyqS~;|8xCUt=g^YUKIw6G?DVYG^qWQf7;ghYP}fy zB&vT+`i*#&)}@@zpB14w57U2;a(YsYL%>ab>K2;v@sD{GA0{Z(RXmN99(vsOFUIZ1 zzc(#^zjS=|>fyszuF^f*4)?=9_6MGR5Rdx9f197e?10Zl>u4M>j_u^Js_AZp0;`2W z&_PApQYc0MRRM5=VffiCnFcj6hEREAP&3Alu^lQ{gw~@u0piY~r9oPP=Cl9|d4u-s z4#WnMg{-|*sqbzj3UDur*AWHm;ab3@jBPw#~290F#Wubx8jACwH zbA(VP8wCfzQRC~Ufjlq-Iv6M_dK(?H1P8^#$m=I`fGDk`(OY0nIu%kdvaV1t_VyeT zJ%l9bbZCFGwYmwuTz>z_8_(&^i5IeExeopv^ZVQke*sZ`@11->`N_7%jK1D4>Ns}% zAaJ!T$s5(Ps@s>o99o<;5}aO%VVI_{{Wdek`7*C>^5%b=#-@JJ^lsYzIdw8r;NSoB zOY?7k*VQ_KznzgyPOT2vz1nRZ*zGf^EqOqs@X z={TXi2C*1UTpikjOhglb@6ZIC4qBA#hK5tO(GoZVG&4Y-o{S_i0a_q{X(|ekgvQel z!1`o%Q(+Ii2`~ihPM#07lx!4(^NWBAp;K|`!%#>kH_l^r zFeMd5^sz)&3dSVTF`(UYocbCv_*8-l_9}w~F37|~rdJge)p5H)T^3NRg z7AwsVWa7tMos%qD*`H+JvKNGt)2F50&@1UZW%v}^`q1>ZQD7to(805xe?-lCqHB!1}4{#Y}GZdXav|lwFwz){8e&Q zrn6D`(fV%W^WV{*5)RPKlXJr^QAN@BvJT$qw%n=zVN$VBTr0gAd{kLnXPS84+ad39 z;DTNL?+Di+wXdrmF5A338jM<&2(#>pjLzn};MewF`)~Grs;7U)e^8N8PFa27H;7Ti zeiaWO$voTlun-hajX$3XrS2FyrC!l$86}3Mo*%Ls*aMD`0aQF}nvQ(R1Kb4?;Y1M7 z2s@K8t#Y<*`9qI@8h#+f!m5T9><`eo`=Giu!ow2o1e+TvR@pRY$;eWyp@o22ksjhf zFr@6B?C;<^;jbg=z$YXrtC9`sCJnVZsSk;XWfiw$fH{TmoOErQ*YYGs4m_C7d7cLd zuG`MZq`LvseQP;hxI9jLg<$<&rK62>j;$2{uSn;!)w4ZU#V_Z8xLD2v*cx9>fJ}?~ z{laajHMO4ZmUb!gnfipmB-=`l(N!UpZdu{^z1KDMyJCvZ)~sjQbetqtS3c3jrm-5n zhVp&5Tb$5Wb8)t-@}n$V$?}KuxjhP4+el_|A9Z>duw$xPa?8rUe!_mbnB;9O$9DP*D{(|P#0y614b9#Z;DRj_L63?zi^)3y=C{2PM*bWAj0zkp!7$OAfkc8lL zwwgCC@M~zBtR>nyhfyc1Cv!IH$>`e(aw}@W6legRo{+ECYIplZ5~u&HjBxO08wq;x zbruu9Umpki~J6pzKT%e$9DfW?yGs4jgCX9oFj&|eRVQNWH8ew;~$MacP zS=~9j3Z7t{%)&=c!7Cq3^zF1^`McR$-yn2^(x>_j!sa}3x~!nIYge(=f80m^@jrr9 z*8nF+YYH6n6)fmNjF;{S%ngJAvRMMD!|SyOhQM4Zwnm9Mi1 zfjcE@VXk80eR7cotBSY5I~Gc9qHMHSH)i#PWYO<@8^hG}hyqm!RSv0MF|`s!F2)7# z*%Qgsi!Iyhv;$Uhzq~r0i8bcdw)Hx(zIqVgp2h9LEKj;M`fQsJ#!s1jDPNS~FZdVt z!eZcYZo+LF-}wkkk9@c`U`kRl%x5`W0{g?xE#RBR%kRxJlaoT#=UMNOFuuGRSD()) z6rz!f@!F!-tvPNiBkBUNRwHRJ5%((H{~?iUGu&$#OIB<|JX0nEo;K@rT7?d)OF(O_fKjw zN&U7=KZNd`34nk2|5J74Z~PTgv@IFuE#=cVb(j#UXfrk%nzP$o5Vu$qBb+Q`MUzrV zl!EB}PHyE~)?K&3=e8_XPF~~tu&YjYb$f1J(Y3;iZB^KCnQ3Z)FF@MsBJ;qO*RE09~;%bIak{e z>-MkUSJfMnrLcZ^N!$DKhg`(NQG0;e{Z#g2obY$&M=ev9Di;cuo#ikzVg_w-~fWvm- zqE-L&!D;XPl`U*O2W{e+*QR5E;GHRG18LD9mAM%oTQ>4!XhWE-@RB618;??xuFC&-o2*3Qe>Pbxv1mG#4~vZKe*g#@YYME35eke348n`WwbgQwsK24 z4;d)}6L{ukI+KDOZ98}@J1h5h5IRf6|KMM&y7Y(tckH(XOV_Zs=q>Tgbqo$s=`z)H3>z-8l1(l!}{E&YY9<*e%Hk0YLmf<;}cwmAZ}6ZWEm^ zsEcE8F8eU#JcaOGIJ*Xn{R;eID+xRF?Xi3SZ*p&5wf8FWn={Kbej;EeTcS}X_rS;^ zzH)$6PndqKGvn|sx1m~E4d02EQ8MmNl`-dZU(eY*PgqyaUCCgh(*smFX(xH0qSG5& zdYUA$OB}_FraU>lboG&&E|$&5m$`YQCh|Mm5+txiLf8uxO^Pwhz%NNvCVrE-nd6R^ z%il#6vqMFN<3>2vxFn-XYF{wjiCwqdI>>n~E1?(`CdK0Xz;NQ?sHO18T`hNHl&2N#yI@1HmYCjT;g z6twl2bQtWbS2lw5zn@H{!i}ypYS>px#?kvu&2Pjwb!Sp3mmf06zHtrX@*6B~0B8xz zOg=RUKT8&4nYpjn)8b$yo0h znXO;WSzPy#gE3VHMq_)4CZlBrIP2D75vEX@>;Bcjsrf+BRO}Jb(a^nXb~^N+YfL-!cHmk4`qIe!eNqN{j6^ z9xs}3li1gGk<{%ce$#leJ34qH5S39LP2nD+C*?fiHuN38rE~ij?~H-F4+m&C`1)SB zgFl%MS6y(;F*1}jwhI_vue%MQrvpj&)jn9h?s4A3mAr17!hJ{HWZ90(@#}2S0WZ_! zi{!Dj{T%G4_-pnw?}vr_4#{=pw(|KT54nXM^}xlBJE`%c*rX=qQ_PS zmPxw&(9kp8S2VH9sHl($iOWllvkN7thAT}kD1&5s~a0AFlss_Xmpb16=o9fa?wEARju9yrj`Hm-_R+!N9 zMDvY)dp;c~Endxm(+v&@MGO?vevpXkfy3L!& z*TBleF5a|>nth75aP&q{iGj+gBz(TNt=M6To|S^6*)gb^SM~fA?n@cj{7k3+-KCvb zs`2vvJ6ErR)z+c;aD7BKKWV@EzNf|k?zV+gPs&?M@i!KVYxxbH<`|wq_s($s(~und z7ZC)GC_Zo5+%ZSG#-vF;a+RgI1gW|-^m3A5=0@O@UZ@z`N`eeysl?(n{ZqE)!S@Zo z=x6Q?B8%iV9TY?(-Bw4{){~%x!O+J8xAjr{cLpH-rgZ)-nf{96i`r=v6RTDYhFs?K z7-v-ZZ-&}>MdPDUdv}8h?6W|;uMhQI4$^9lIQbek|MqL9s?559ptA2xEqv#^?_nUE zB-hyg<^EX?_!Iw~hE<0FpV4c|15PxI1UIx7v;r~*-NMxNHhh~c`}pk}@u>4Mwd?2h!#_z^naaLM?mMVzQor;rv>D~! z+|V^`{XBN{ij8H)n#A)(<12;gjMtwxQc9dm9NX9Ctsc?42Ppo&V#Tp+9Y(orVB08= zaquZg&Q5dQpWgG0f(>ZIK3gVkt;(U@k%9e24lu9YoFC(-0PoRj>Z(`KhDIzgyDWfRQE;u; zSw9*<8z+c9f{$mIU!euW3AW+unlwly5|+ZY69Oww5j_*4D*2n3gN2?3-)5~kM|cNv z=@H2!r~ga58kS2oW76rCmqV-Hl!LN>-Uqq>*&@{Qhtp~MhWTZs67`2}^y#wFeFAw2jSFcPjVeiO)!DS3cCT=`9+2V@J## z8%HNs{TD{}XN7uL8Euo3u49*9znfT%;<1V{zh<-GYD;QRoBpQh9%#a9oc}quO}*-u zj-AeVg0o`Ilgk#I6+NYs7KioaztK$cYI7zD;`+K0;mpANV4hROpXkh@)E)Y;30MteZ7SN&<7|^&g%LD9y4@m2i zw2L59?9mt&TpwVACc{2qWjNcO)wUr8KC*F*GVxMF@Xc|aW||`2C12BBF|oC@_x_M> z#s$-8`M+Lf_XkvcarW1Wzlg{)ToYz$$ac3oQm9EfNWL)pg8IXx>bCaB%!VbeGHNu_ z;EFcgX0#>$Tegk~f{tawtVrlDn|?L3+{sR!e67be)%XSMOFNU5ZGMlRb|a0!#fPZi zK=I)fk7T=NqF=;F$o8b3e=&u8c8OImvaZ={PsLoJvwLOKk1}0c`oL;VcReoi^0zQ0 zeDH7llplM#;DET?gbqtg$CkK-0IlFB&TXPI=xJ0rZE&a@1W2e1F+`Rx#Mn!MT6kk@ zydL;ym<$L?FoM4p8g|*X-_u5;EG9iI{gZCUC1}rW=-;1ExstVGIrh4xOSRsNjoP6+ z@tpXF|8@te9e?oeE>%!?B1)So45EQ}0JL<#T7o_Fu>gQD3`7fTrJ5rL$tGH8BCm%V zwWkFK1q_kF`e|YCdk7($Y(DK>^gbaY#cfPEKW>RO#_v2v_el7asb2!KB1}9=L{Fo$vK)HM%XaGn-Vz3G zl_~UH%Ag3ypowkVBl3B2dJKiM{hR-e__Mn_y8>$}PQwdUI%*t@p`qwZm=U6Jl#aFlZ{bcINP~>BGb1zLMBLTn zWRZa*pBreLTWO0Rj2YzJojq&=5oyB~A0PYiaG_>kYXz4Un_*xzt$NKT%CjY{u{Sa1jeA5+AcPmMDx=~8ab2DkC_$@>aDXP+j zA94&+LkbLEL=0{?GR2kE{tA|$-4s0s)5^H7>_A((b}qgQrY*$34Jn}vgWI#Pmv|Czg2XPr%XwJbP7xt95A323U8$t;!!vT85+fT(~k#dj$ zbM|+l9zR85H{@t_bUwbkFp_%qtx2C~7$*?q$sV)2qi1zSfI zK{|ei8J~KS-O8XTYWaj<~`v5fw%P5OX&EjD7$2{Q1B2m#%6t;HAJC){*UgH8vAOlL;#SOa;ZR%z6(T*- zQLG7!WPnKaI3-O<-LlY?j8O=KpO%iE+ZeG-AB0~TX4Jl!JBIeit&1(VmsR%C{LT>H z3nsSHi%x_EO(mSEFTPt>Qv1@Zm4ZU%Q1L?nx5bG?l@Fqodlq)d!U4n-2Gu@Wrmdy; zB8e8mGe4tc+(OXp@<7=}ns0K1V!cmN!OE3^{RkaKkptr;T>CP8A2x#!YPKW zLaoMSqdKO-FMSZ9y_5E^UZVIYcd^`FM{PDl3US*sn<#4F6IwWy5d$c{sAXj`MNT-M zD5>VHHgl45i)btWnWFu!PiI)H%#xO5l<}^Xp)De-4CdCa8fi0uZDlewj4uImTF=W} z9D9eAv~f*sul$lqLPVx&a4l)QOyqk_7b5ZY-+c$@f2J`1;NNjp|Gay-ralZtJ4x!) z#ciNcxqu)qU~E~C3>5)~K_O%S0L%$s*l>#9;{mXhY0o$L*C4N^qu+3RbP-Jeq6DFC zJzm0$LqFfi?iOqdw0bZF%hOW>!9-+?AQO%%5NoJ4@6(=Z5n52EJsAAhy4(uoE2g0@ zX%?9s!sI%($}8x87e`AR7X_^-)0CpY0M3muXu-KbC3w))TXuJngf!$5+cr%#SD!J? zOfUAzKaQ?+lH`ITu+Lvs>I$>ajd+N}Nz1bqMg~<9qpg#JVNN^dnUN}F}GpUg>PYvZ9B$tXn%stJ|N#}EdH+zU&+T@1?-jR<|JZ|10 zHFY*6>~~tU-BFotb1|D(`6250vw#DlrF-KX-Pyq7zpST!e!(C79ZywT{=PqUVP$-x z`h-6#wGWNnld*!u;{au3ZTDdt3Y4A;ffg*=J0j=Ybkpn69k%)lVYq0$PP37pgm|wE z0fY$$_q+(vouJ7nf@5jDO-iV)xf@Wzj%$+ojc7sJ^}AU8YyVOUw-=rWf6gGE*ZwU) zIo^OLV3XT@s3TrByZ+{BQyTCJkk zS-mw{)ekRM#GVcW3a|RnyR1s^cRuYy`h|6LNB>ogdKmap+x}fdvVY-y#V)349{R$u z=WLb78Z&-8{!cpiK96|w8_ACUWJX5w3=x9X6Ct_n%0h*bIvwlz-(3yIDYaId{G5VR z`xlS^DS@hVb<|B?_P@F#?H%{_v0Zz!&R@J}B3&fDwJCmDo8=FG+oE=$H6OP=K9URz z{?Gj%ho7oFf7d@3QxO}9aah2Nuq}F=IH@D*fiZM4gcDoUE+TOn#nlXu0yqp#FP}lc zgXzHJIX(aYLm0~+M?yCcP(R0RI~mj@b1myNMbUVipL8wjX(SJiclmW4POw_@M(4^j zD}M*hd8d!^yxH2mR>HmL{xt|HhM0F@M`95k?Z7J+-{n&@)E4IM&RV@X6wSVAIbigO zvCl+{?X~bRvyfH?c09<7<1(pp+#g;=H`YBOWYeJdN?i;Z>A^Xi++S$nI4wdbDVwCo z<`3;`OWg5JzsgSnC6v^Dlz`GPNsw5@@S(8CP>}>#moUDXj2%I7yRT>$*rykZq)_(LLpg7hCFs0&<}K|c7GBw*gNuB( zw`ZR(9d4LAO1Udp5@;&{FU?$UIR{C+u&X2aA5{a>FEGbo|9of zn}#_thT2XK83I5Vp@JGzLC7Cab z;-qWlzCmsanQYM&2w#HqJD99W6?_-(NiO1SYSQhWDQ7JXgCt{XxHH6#?Q`R84u%MQrFdW(2?#?A7f7O%C}?o*l~C@uUZx%i|v*yj(i dj=A%*)^1Bl_l&Kz)Qrlcy#BuZ@XrR&{{Y7O)$#xU literal 0 HcmV?d00001 diff --git a/js/testapps/compat-oai/src/index.ts b/js/testapps/compat-oai/src/index.ts index d5ce6ca8da..506eebb611 100644 --- a/js/testapps/compat-oai/src/index.ts +++ b/js/testapps/compat-oai/src/index.ts @@ -377,6 +377,25 @@ ai.defineFlow('transcribe', async () => { return text; }); +// translation sample +ai.defineFlow('translate', async () => { + const audioFile = fs.readFileSync('audio-korean.mp3'); + + const { text } = await ai.generate({ + model: openAI.model('whisper-1-translate'), + prompt: [ + { + media: { + contentType: 'audio/mp3', + url: `data:audio/mp3;base64,${audioFile.toString('base64')}`, + }, + }, + ], + }); + + return text; +}); + // PDF file input example ai.defineFlow( { From abdee5e7fc106ac608fa09b92e9ab1cf6dd86eac Mon Sep 17 00:00:00 2001 From: 7hokerz Date: Sat, 21 Feb 2026 13:03:24 +0900 Subject: [PATCH 2/4] refactor: moved translation logic to translate.ts --- js/plugins/compat-oai/src/audio.ts | 192 --------------- js/plugins/compat-oai/src/index.ts | 10 +- js/plugins/compat-oai/src/openai/index.ts | 6 +- .../compat-oai/src/openai/translation.ts | 2 +- js/plugins/compat-oai/src/translate.ts | 223 ++++++++++++++++++ 5 files changed, 234 insertions(+), 199 deletions(-) create mode 100644 js/plugins/compat-oai/src/translate.ts diff --git a/js/plugins/compat-oai/src/audio.ts b/js/plugins/compat-oai/src/audio.ts index 48b1b7502f..c0ce67768a 100644 --- a/js/plugins/compat-oai/src/audio.ts +++ b/js/plugins/compat-oai/src/audio.ts @@ -28,8 +28,6 @@ import type { SpeechCreateParams, Transcription, TranscriptionCreateParams, - TranslationCreateParams, - TranslationCreateResponse, } from 'openai/resources/audio/index.mjs'; import { PluginOptions } from './index.js'; import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; @@ -42,10 +40,6 @@ export type TranscriptionRequestBuilder = ( req: GenerateRequest, params: TranscriptionCreateParams ) => void; -export type TranslationRequestBuilder = ( - req: GenerateRequest, - params: TranslationCreateParams -) => void; export const TRANSCRIPTION_MODEL_INFO: ModelInfo = { supports: { @@ -67,16 +61,6 @@ export const SPEECH_MODEL_INFO: ModelInfo = { }, }; -export const TRANSLATION_MODEL_INFO: ModelInfo = { - supports: { - media: true, - output: ['text', 'json'], - multiturn: false, - systemRole: false, - tools: false, - }, -}; - const ChunkingStrategySchema = z.object({ type: z.string(), prefix_padding_ms: z.number().int().optional(), @@ -108,14 +92,6 @@ export const SpeechConfigSchema = z.object({ .optional(), }); -export const TranslationConfigSchema = GenerationCommonConfigSchema.pick({ - temperature: true, -}).extend({ - response_format: z - .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) - .optional(), -}); - /** * Supported media formats for Audio generation */ @@ -444,171 +420,3 @@ export function compatOaiTranscriptionModelRef< namespace, }); } - -function toTranslationRequest( - modelName: string, - request: GenerateRequest, - requestBuilder?: TranslationRequestBuilder -): TranslationCreateParams { - const message = new Message(request.messages[0]); - const media = message.media; - if (!media?.url) { - throw new Error('No media found in the request'); - } - const mediaBuffer = Buffer.from( - media.url.slice(media.url.indexOf(',') + 1), - 'base64' - ); - const mediaFile = new File([mediaBuffer], 'input', { - type: - media.contentType ?? - media.url.slice('data:'.length, media.url.indexOf(';')), - }); - const { - temperature, - version: modelVersion, - maxOutputTokens, - stopSequences, - topK, - topP, - ...restOfConfig - } = request.config ?? {}; - - let options: TranslationCreateParams = { - model: modelVersion ?? modelName, - file: mediaFile, - prompt: message.text, - temperature, - }; - if (requestBuilder) { - requestBuilder(request, options); - } else { - options = { - ...options, - ...restOfConfig, // passthrough rest of the config - }; - } - const outputFormat = request.output?.format as 'json' | 'text' | 'media'; - const customFormat = request.config?.response_format; - if (outputFormat && customFormat) { - if ( - outputFormat === 'json' && - customFormat !== 'json' && - customFormat !== 'verbose_json' - ) { - throw new Error( - `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` - ); - } - } - if (outputFormat === 'media') { - throw new Error(`Output format ${outputFormat} is not supported.`); - } - options.response_format = customFormat || outputFormat || 'text'; - for (const k in options) { - if (options[k] === undefined) { - delete options[k]; - } - } - return options; -} - -function translationToGenerateResponse( - result: TranslationCreateResponse | string -): GenerateResponseData { - return { - message: { - role: 'model', - content: [ - { - text: typeof result === 'string' ? result : result.text, - }, - ], - }, - finishReason: 'stop', - raw: result, - }; -} - -/** - * Method to define a new Genkit Model that is compatible with Open AI - * Translation API. - * - * These models are to be used to translate audio to text. - * - * @param params An object containing parameters for defining the OpenAI - * translation model. - * @param params.ai The Genkit AI instance. - * @param params.name The name of the model. - * @param params.client The OpenAI client instance. - * @param params.modelRef Optional reference to the model's configuration and - * custom options. - * - * @returns the created {@link ModelAction} - */ -export function defineCompatOpenAITranslationModel< - CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, ->(params: { - name: string; - client: OpenAI; - pluginOptions?: PluginOptions; - modelRef?: ModelReference; - requestBuilder?: TranslationRequestBuilder; -}) { - const { - name, - client: defaultClient, - pluginOptions, - modelRef, - requestBuilder, - } = params; - const modelName = toModelName(name, pluginOptions?.name); - const actionName = `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; - - return model( - { - name: actionName, - ...modelRef?.info, - configSchema: modelRef?.configSchema, - }, - async (request, { abortSignal }) => { - const params = toTranslationRequest(modelName, request, requestBuilder); - const client = maybeCreateRequestScopedOpenAIClient( - pluginOptions, - request, - defaultClient - ); - const result = await client.audio.translations.create(params, { - signal: abortSignal, - }); - return translationToGenerateResponse(result); - } - ); -} - -/** Translation ModelRef helper, with reasonable defaults for - * OpenAI-compatible providers */ -export function compatOaiTranslationModelRef< - CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, ->(params: { - name: string; - info?: ModelInfo; - configSchema?: CustomOptions; - config?: any; - namespace?: string; -}) { - const { - name, - info = TRANSLATION_MODEL_INFO, - configSchema, - config = undefined, - namespace, - } = params; - return modelRef({ - name, - configSchema: configSchema || (TranslationConfigSchema as any), - info, - config, - namespace, - }); -} diff --git a/js/plugins/compat-oai/src/index.ts b/js/plugins/compat-oai/src/index.ts index 55e1f3ac8c..f474ebbf9d 100644 --- a/js/plugins/compat-oai/src/index.ts +++ b/js/plugins/compat-oai/src/index.ts @@ -24,16 +24,12 @@ import { toModelName } from './utils.js'; export { SpeechConfigSchema, TranscriptionConfigSchema, - TranslationConfigSchema, compatOaiSpeechModelRef, compatOaiTranscriptionModelRef, - compatOaiTranslationModelRef, defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, - defineCompatOpenAITranslationModel, type SpeechRequestBuilder, type TranscriptionRequestBuilder, - type TranslationRequestBuilder, } from './audio.js'; export { defineCompatOpenAIEmbedder } from './embedder.js'; export { @@ -49,6 +45,12 @@ export { openAIModelRunner, type ModelRequestBuilder, } from './model.js'; +export { + TranslationConfigSchema, + compatOaiTranslationModelRef, + defineCompatOpenAITranslationModel, + type TranslationRequestBuilder, +} from './translate.js'; export interface PluginOptions extends Partial> { apiKey?: ClientOptions['apiKey'] | false; diff --git a/js/plugins/compat-oai/src/openai/index.ts b/js/plugins/compat-oai/src/openai/index.ts index 20c0fa2a73..3d1c22baf1 100644 --- a/js/plugins/compat-oai/src/openai/index.ts +++ b/js/plugins/compat-oai/src/openai/index.ts @@ -30,10 +30,8 @@ import OpenAI from 'openai'; import { defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, - defineCompatOpenAITranslationModel, SpeechConfigSchema, TranscriptionConfigSchema, - TranslationConfigSchema, } from '../audio.js'; import { defineCompatOpenAIEmbedder } from '../embedder.js'; import { @@ -42,6 +40,10 @@ import { } from '../image.js'; import { openAICompatible, PluginOptions } from '../index.js'; import { defineCompatOpenAIModel } from '../model.js'; +import { + defineCompatOpenAITranslationModel, + TranslationConfigSchema, +} from '../translate.js'; import { gptImage1RequestBuilder, openAIImageModelRef, diff --git a/js/plugins/compat-oai/src/openai/translation.ts b/js/plugins/compat-oai/src/openai/translation.ts index 5cc76b2ec2..2dec9ac4f2 100644 --- a/js/plugins/compat-oai/src/openai/translation.ts +++ b/js/plugins/compat-oai/src/openai/translation.ts @@ -17,7 +17,7 @@ import { z } from 'genkit'; import { ModelInfo } from 'genkit/model'; -import { compatOaiTranslationModelRef } from '../audio'; +import { compatOaiTranslationModelRef } from '../translate'; /** OpenAI translation ModelRef helper, same as the OpenAI-compatible spec. */ export function openAITranslationModelRef< diff --git a/js/plugins/compat-oai/src/translate.ts b/js/plugins/compat-oai/src/translate.ts new file mode 100644 index 0000000000..6910700904 --- /dev/null +++ b/js/plugins/compat-oai/src/translate.ts @@ -0,0 +1,223 @@ +/** + * Copyright 2024 The Fire Company + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + GenerateRequest, + GenerateResponseData, + ModelReference, +} from 'genkit'; +import { GenerationCommonConfigSchema, Message, modelRef, z } from 'genkit'; +import type { ModelAction, ModelInfo } from 'genkit/model'; +import { model } from 'genkit/plugin'; +import OpenAI from 'openai'; +import type { + TranslationCreateParams, + TranslationCreateResponse, +} from 'openai/resources/audio/index.mjs'; +import { PluginOptions } from './index.js'; +import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; + +export type TranslationRequestBuilder = ( + req: GenerateRequest, + params: TranslationCreateParams +) => void; + +export const TRANSLATION_MODEL_INFO: ModelInfo = { + supports: { + media: true, + output: ['text', 'json'], + multiturn: false, + systemRole: false, + tools: false, + }, +}; + +export const TranslationConfigSchema = GenerationCommonConfigSchema.pick({ + temperature: true, +}).extend({ + response_format: z + .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) + .optional(), +}); + +function toTranslationRequest( + modelName: string, + request: GenerateRequest, + requestBuilder?: TranslationRequestBuilder +): TranslationCreateParams { + const message = new Message(request.messages[0]); + const media = message.media; + if (!media?.url) { + throw new Error('No media found in the request'); + } + const mediaBuffer = Buffer.from( + media.url.slice(media.url.indexOf(',') + 1), + 'base64' + ); + const mediaFile = new File([mediaBuffer], 'input', { + type: + media.contentType ?? + media.url.slice('data:'.length, media.url.indexOf(';')), + }); + const { + temperature, + version: modelVersion, + maxOutputTokens, + stopSequences, + topK, + topP, + ...restOfConfig + } = request.config ?? {}; + + let options: TranslationCreateParams = { + model: modelVersion ?? modelName, + file: mediaFile, + prompt: message.text, + temperature, + }; + if (requestBuilder) { + requestBuilder(request, options); + } else { + options = { + ...options, + ...restOfConfig, // passthrough rest of the config + }; + } + const outputFormat = request.output?.format as 'json' | 'text' | 'media'; + const customFormat = request.config?.response_format; + if (outputFormat && customFormat) { + if ( + outputFormat === 'json' && + customFormat !== 'json' && + customFormat !== 'verbose_json' + ) { + throw new Error( + `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` + ); + } + } + if (outputFormat === 'media') { + throw new Error(`Output format ${outputFormat} is not supported.`); + } + options.response_format = customFormat || outputFormat || 'text'; + for (const k in options) { + if (options[k] === undefined) { + delete options[k]; + } + } + return options; +} + +function translationToGenerateResponse( + result: TranslationCreateResponse | string +): GenerateResponseData { + return { + message: { + role: 'model', + content: [ + { + text: typeof result === 'string' ? result : result.text, + }, + ], + }, + finishReason: 'stop', + raw: result, + }; +} + +/** + * Method to define a new Genkit Model that is compatible with Open AI + * Translation API. + * + * These models are to be used to translate audio to text. + * + * @param params An object containing parameters for defining the OpenAI + * translation model. + * @param params.ai The Genkit AI instance. + * @param params.name The name of the model. + * @param params.client The OpenAI client instance. + * @param params.modelRef Optional reference to the model's configuration and + * custom options. + * + * @returns the created {@link ModelAction} + */ +export function defineCompatOpenAITranslationModel< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + client: OpenAI; + pluginOptions?: PluginOptions; + modelRef?: ModelReference; + requestBuilder?: TranslationRequestBuilder; +}) { + const { + name, + client: defaultClient, + pluginOptions, + modelRef, + requestBuilder, + } = params; + const modelName = toModelName(name, pluginOptions?.name); + const actionName = `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; + + return model( + { + name: actionName, + ...modelRef?.info, + configSchema: modelRef?.configSchema, + }, + async (request, { abortSignal }) => { + const params = toTranslationRequest(modelName, request, requestBuilder); + const client = maybeCreateRequestScopedOpenAIClient( + pluginOptions, + request, + defaultClient + ); + const result = await client.audio.translations.create(params, { + signal: abortSignal, + }); + return translationToGenerateResponse(result); + } + ); +} + +/** Translation ModelRef helper, with reasonable defaults for + * OpenAI-compatible providers */ +export function compatOaiTranslationModelRef< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + info?: ModelInfo; + configSchema?: CustomOptions; + config?: any; + namespace?: string; +}) { + const { + name, + info = TRANSLATION_MODEL_INFO, + configSchema, + config = undefined, + namespace, + } = params; + return modelRef({ + name, + configSchema: configSchema || (TranslationConfigSchema as any), + info, + config, + namespace, + }); +} From b4a70643a75f2bd4870c58ec9f8659be73ad09c0 Mon Sep 17 00:00:00 2001 From: 7hokerz Date: Tue, 24 Feb 2026 19:33:35 +0900 Subject: [PATCH 3/4] feat: implement Whisper-specific configuration and API handlers --- js/plugins/compat-oai/src/audio.ts | 212 +++++++++++++++++ js/plugins/compat-oai/src/index.ts | 10 +- js/plugins/compat-oai/src/openai/index.ts | 67 ++---- js/plugins/compat-oai/src/openai/stt.ts | 3 - .../src/openai/{translation.ts => whisper.ts} | 21 +- js/plugins/compat-oai/src/translate.ts | 223 ------------------ js/testapps/compat-oai/src/index.ts | 4 +- 7 files changed, 246 insertions(+), 294 deletions(-) rename js/plugins/compat-oai/src/openai/{translation.ts => whisper.ts} (56%) delete mode 100644 js/plugins/compat-oai/src/translate.ts diff --git a/js/plugins/compat-oai/src/audio.ts b/js/plugins/compat-oai/src/audio.ts index c0ce67768a..0a9dbbd25a 100644 --- a/js/plugins/compat-oai/src/audio.ts +++ b/js/plugins/compat-oai/src/audio.ts @@ -28,6 +28,8 @@ import type { SpeechCreateParams, Transcription, TranscriptionCreateParams, + TranslationCreateParams, + TranslationCreateResponse, } from 'openai/resources/audio/index.mjs'; import { PluginOptions } from './index.js'; import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; @@ -40,6 +42,10 @@ export type TranscriptionRequestBuilder = ( req: GenerateRequest, params: TranscriptionCreateParams ) => void; +export type TranslationRequestBuilder = ( + req: GenerateRequest, + params: TranslationCreateParams +) => void; export const TRANSCRIPTION_MODEL_INFO: ModelInfo = { supports: { @@ -61,6 +67,16 @@ export const SPEECH_MODEL_INFO: ModelInfo = { }, }; +export const WHISPER_MODER_INFO: ModelInfo = { + supports: { + media: true, + output: ['text', 'json'], + multiturn: false, + systemRole: false, + tools: false, + }, +}; + const ChunkingStrategySchema = z.object({ type: z.string(), prefix_padding_ms: z.number().int().optional(), @@ -92,6 +108,19 @@ export const SpeechConfigSchema = z.object({ .optional(), }); +export const WhisperConfigSchema = GenerationCommonConfigSchema.pick({ + temperature: true, +}).extend({ + /** When true, uses Translation API. Default: false **/ + translate: z.boolean().optional().default(false), + response_format: z + .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) + .optional(), + // transcription-only fields (ignored when translate=true) + language: z.string().optional(), + timestamp_granularities: z.array(z.enum(['word', 'segment'])).optional(), +}); + /** * Supported media formats for Audio generation */ @@ -420,3 +449,186 @@ export function compatOaiTranscriptionModelRef< namespace, }); } + +function toTranslationRequest( + modelName: string, + request: GenerateRequest, + requestBuilder?: TranslationRequestBuilder +): TranslationCreateParams { + const message = new Message(request.messages[0]); + const media = message.media; + if (!media?.url) { + throw new Error('No media found in the request'); + } + const mediaBuffer = Buffer.from( + media.url.slice(media.url.indexOf(',') + 1), + 'base64' + ); + const mediaFile = new File([mediaBuffer], 'input', { + type: + media.contentType ?? + media.url.slice('data:'.length, media.url.indexOf(';')), + }); + const { + temperature, + version: modelVersion, + maxOutputTokens, + stopSequences, + topK, + topP, + ...restOfConfig + } = request.config ?? {}; + + let options: TranslationCreateParams = { + model: modelVersion ?? modelName, + file: mediaFile, + prompt: message.text, + temperature, + }; + if (requestBuilder) { + requestBuilder(request, options); + } else { + options = { + ...options, + ...restOfConfig, // passthrough rest of the config + }; + } + const outputFormat = request.output?.format as 'json' | 'text' | 'media'; + const customFormat = request.config?.response_format; + if (outputFormat && customFormat) { + if ( + outputFormat === 'json' && + customFormat !== 'json' && + customFormat !== 'verbose_json' + ) { + throw new Error( + `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` + ); + } + } + if (outputFormat === 'media') { + throw new Error(`Output format ${outputFormat} is not supported.`); + } + options.response_format = customFormat || outputFormat || 'text'; + for (const k in options) { + if (options[k] === undefined) { + delete options[k]; + } + } + return options; +} + +function translationToGenerateResponse( + result: TranslationCreateResponse | string +): GenerateResponseData { + return { + message: { + role: 'model', + content: [ + { + text: typeof result === 'string' ? result : result.text, + }, + ], + }, + finishReason: 'stop', + raw: result, + }; +} + +/** + * Method to define a new Genkit Model that is compatible with Open AI + * Whisper API. + * + * These models are to be used to transcribe or translate audio to text. + * + * @param params An object containing parameters for defining the OpenAI + * whisper model. + * @param params.ai The Genkit AI instance. + * @param params.name The name of the model. + * @param params.client The OpenAI client instance. + * @param params.modelRef Optional reference to the model's configuration and + * custom options. + * + * @returns the created {@link ModelAction} + */ +export function defineCompatOpenAIWhisperModel(params: { + name: string; + client: OpenAI; + pluginOptions?: PluginOptions; + modelRef?: ModelReference; + requestBuilder?: TranscriptionRequestBuilder | TranslationRequestBuilder; +}) { + const { + name, + pluginOptions, + client: defaultClient, + modelRef, + requestBuilder, + } = params; + const modelName = toModelName(name, pluginOptions?.name); + const actionName = + modelRef?.name ?? `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; + + return model( + { + name: actionName, + ...modelRef?.info, + configSchema: modelRef?.configSchema, + }, + async (request, { abortSignal }) => { + const isTranslate = request.config?.translate === true; + const client = maybeCreateRequestScopedOpenAIClient( + pluginOptions, + request, + defaultClient + ); + + if (isTranslate) { + // Translation API + const params = toTranslationRequest(modelName, request, requestBuilder); + const result = await client.audio.translations.create(params, { + signal: abortSignal, + }); + return translationToGenerateResponse(result); + } else { + // Transcription API + const params = toSttRequest(modelName, request, requestBuilder); + const result = await client.audio.transcriptions.create( + { + ...params, + stream: false, + }, + { signal: abortSignal } + ); + return transcriptionToGenerateResponse(result); + } + } + ); +} + +/** Whisper ModelRef helper, with reasonable defaults for + * OpenAI-compatible providers */ +export function compatOaiWhisperModelRef< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + info?: ModelInfo; + configSchema?: CustomOptions; + config?: any; + namespace?: string; +}) { + const { + name, + info = WHISPER_MODER_INFO, + configSchema, + config = undefined, + namespace, + } = params; + return modelRef({ + name, + configSchema: configSchema || (WhisperConfigSchema as any), + info, + config, + namespace, + }); +} diff --git a/js/plugins/compat-oai/src/index.ts b/js/plugins/compat-oai/src/index.ts index f474ebbf9d..19198d3496 100644 --- a/js/plugins/compat-oai/src/index.ts +++ b/js/plugins/compat-oai/src/index.ts @@ -24,12 +24,16 @@ import { toModelName } from './utils.js'; export { SpeechConfigSchema, TranscriptionConfigSchema, + WhisperConfigSchema, compatOaiSpeechModelRef, compatOaiTranscriptionModelRef, + compatOaiWhisperModelRef, defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, + defineCompatOpenAIWhisperModel, type SpeechRequestBuilder, type TranscriptionRequestBuilder, + type TranslationRequestBuilder, } from './audio.js'; export { defineCompatOpenAIEmbedder } from './embedder.js'; export { @@ -45,12 +49,6 @@ export { openAIModelRunner, type ModelRequestBuilder, } from './model.js'; -export { - TranslationConfigSchema, - compatOaiTranslationModelRef, - defineCompatOpenAITranslationModel, - type TranslationRequestBuilder, -} from './translate.js'; export interface PluginOptions extends Partial> { apiKey?: ClientOptions['apiKey'] | false; diff --git a/js/plugins/compat-oai/src/openai/index.ts b/js/plugins/compat-oai/src/openai/index.ts index 3d1c22baf1..29efb71d9d 100644 --- a/js/plugins/compat-oai/src/openai/index.ts +++ b/js/plugins/compat-oai/src/openai/index.ts @@ -30,8 +30,10 @@ import OpenAI from 'openai'; import { defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, + defineCompatOpenAIWhisperModel, SpeechConfigSchema, TranscriptionConfigSchema, + WhisperConfigSchema, } from '../audio.js'; import { defineCompatOpenAIEmbedder } from '../embedder.js'; import { @@ -40,10 +42,6 @@ import { } from '../image.js'; import { openAICompatible, PluginOptions } from '../index.js'; import { defineCompatOpenAIModel } from '../model.js'; -import { - defineCompatOpenAITranslationModel, - TranslationConfigSchema, -} from '../translate.js'; import { gptImage1RequestBuilder, openAIImageModelRef, @@ -59,11 +57,8 @@ import { SUPPORTED_GPT_MODELS, } from './gpt.js'; import { openAITranscriptionModelRef, SUPPORTED_STT_MODELS } from './stt.js'; -import { - openAITranslationModelRef, - SUPPORTED_TRANSLATION_MODELS, -} from './translation.js'; import { openAISpeechModelRef, SUPPORTED_TTS_MODELS } from './tts.js'; +import { openAIWhisperModelRef, SUPPORTED_WHISPER_MODELS } from './whisper.js'; export type OpenAIPluginOptions = Omit; @@ -96,23 +91,15 @@ function createResolver(pluginOptions: PluginOptions) { pluginOptions, modelRef, }); - } else if (actionName.includes('translate')) { - const modelRef = openAITranslationModelRef({ name: actionName }); - return defineCompatOpenAITranslationModel({ + } else if (actionName.includes('whisper')) { + const modelRef = openAIWhisperModelRef({ name: actionName }); + return defineCompatOpenAIWhisperModel({ name: modelRef.name, client, pluginOptions, modelRef, - requestBuilder: (req, params) => { - if (modelRef.name.endsWith('whisper-1-translate')) { - params.model = 'whisper-1'; - } - }, }); - } else if ( - actionName.includes('whisper') || - actionName.includes('transcribe') - ) { + } else if (actionName.includes('transcribe')) { const modelRef = openAITranscriptionModelRef({ name: actionName, }); @@ -168,19 +155,16 @@ const listActions = async (client: OpenAI): Promise => { info: modelRef.info, configSchema: modelRef.configSchema, }); - } else if (model.id.includes('translate')) { + } else if (model.id.includes('whisper')) { const modelRef = - SUPPORTED_TRANSLATION_MODELS[model.id] ?? - openAITranslationModelRef({ name: model.id }); + SUPPORTED_WHISPER_MODELS[model.id] ?? + openAIWhisperModelRef({ name: model.id }); return modelActionMetadata({ name: modelRef.name, info: modelRef.info, configSchema: modelRef.configSchema, }); - } else if ( - model.id.includes('whisper') || - model.id.includes('transcribe') - ) { + } else if (model.id.includes('transcribe')) { const modelRef = SUPPORTED_STT_MODELS[model.id] ?? openAITranscriptionModelRef({ name: model.id }); @@ -240,17 +224,12 @@ export function openAIPlugin(options?: OpenAIPluginOptions): GenkitPluginV2 { ) ); models.push( - ...Object.values(SUPPORTED_TRANSLATION_MODELS).map((modelRef) => - defineCompatOpenAITranslationModel({ + ...Object.values(SUPPORTED_WHISPER_MODELS).map((modelRef) => + defineCompatOpenAIWhisperModel({ name: modelRef.name, client, pluginOptions, modelRef, - requestBuilder: (req, params) => { - if (modelRef.name.endsWith('whisper-1-translate')) { - params.model = 'whisper-1'; - } - }, }) ) ); @@ -301,17 +280,11 @@ export type OpenAIPlugin = { config?: z.infer ): ModelReference; model( - name: - | keyof typeof SUPPORTED_TRANSLATION_MODELS - | (`whisper-${string}-translate` & {}) - | (`${string}-translate` & {}), - config?: z.infer - ): ModelReference; + name: keyof typeof SUPPORTED_WHISPER_MODELS | (`whisper-${string}` & {}), + config?: z.infer + ): ModelReference; model( - name: - | keyof typeof SUPPORTED_STT_MODELS - | (`whisper-${string}` & {}) - | (`${string}-transcribe` & {}), + name: keyof typeof SUPPORTED_STT_MODELS | (`${string}-transcribe` & {}), config?: z.infer ): ModelReference; model( @@ -344,13 +317,13 @@ const model = ((name: string, config?: any): ModelReference => { config, }); } - if (name.includes('translate')) { - return openAITranslationModelRef({ + if (name.includes('whisper')) { + return openAIWhisperModelRef({ name, config, }); } - if (name.includes('whisper') || name.includes('transcribe')) { + if (name.includes('transcribe')) { return openAITranscriptionModelRef({ name, config, diff --git a/js/plugins/compat-oai/src/openai/stt.ts b/js/plugins/compat-oai/src/openai/stt.ts index 1833abaf27..081678cd34 100644 --- a/js/plugins/compat-oai/src/openai/stt.ts +++ b/js/plugins/compat-oai/src/openai/stt.ts @@ -38,7 +38,4 @@ export const SUPPORTED_STT_MODELS = { 'gpt-4o-mini-transcribe': openAITranscriptionModelRef({ name: 'gpt-4o-mini-transcribe', }), - 'whisper-1': openAITranscriptionModelRef({ - name: 'whisper-1', - }), }; diff --git a/js/plugins/compat-oai/src/openai/translation.ts b/js/plugins/compat-oai/src/openai/whisper.ts similarity index 56% rename from js/plugins/compat-oai/src/openai/translation.ts rename to js/plugins/compat-oai/src/openai/whisper.ts index 2dec9ac4f2..5ea1952ab1 100644 --- a/js/plugins/compat-oai/src/openai/translation.ts +++ b/js/plugins/compat-oai/src/openai/whisper.ts @@ -17,10 +17,10 @@ import { z } from 'genkit'; import { ModelInfo } from 'genkit/model'; -import { compatOaiTranslationModelRef } from '../translate'; +import { compatOaiWhisperModelRef } from '../audio'; -/** OpenAI translation ModelRef helper, same as the OpenAI-compatible spec. */ -export function openAITranslationModelRef< +/** OpenAI whisper ModelRef helper, same as the OpenAI-compatible spec. */ +export function openAIWhisperModelRef< CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, >(params: { name: string; @@ -28,18 +28,11 @@ export function openAITranslationModelRef< configSchema?: CustomOptions; config?: any; }) { - return compatOaiTranslationModelRef({ ...params, namespace: 'openai' }); + return compatOaiWhisperModelRef({ ...params, namespace: 'openai' }); } -export const SUPPORTED_TRANSLATION_MODELS = { - /** - * Whisper 1 translation model. - * - * The actual OpenAI model ID is 'whisper-1', but we use 'whisper-1-translate' - * to distinguish it from the 'whisper-1' transcription model. The model ID - * is overridden in index.ts to 'whisper-1' when calling the OpenAI API. - */ - 'whisper-1-translate': openAITranslationModelRef({ - name: 'whisper-1-translate', +export const SUPPORTED_WHISPER_MODELS = { + 'whisper-1': openAIWhisperModelRef({ + name: 'whisper-1', }), }; diff --git a/js/plugins/compat-oai/src/translate.ts b/js/plugins/compat-oai/src/translate.ts deleted file mode 100644 index 6910700904..0000000000 --- a/js/plugins/compat-oai/src/translate.ts +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Copyright 2024 The Fire Company - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { - GenerateRequest, - GenerateResponseData, - ModelReference, -} from 'genkit'; -import { GenerationCommonConfigSchema, Message, modelRef, z } from 'genkit'; -import type { ModelAction, ModelInfo } from 'genkit/model'; -import { model } from 'genkit/plugin'; -import OpenAI from 'openai'; -import type { - TranslationCreateParams, - TranslationCreateResponse, -} from 'openai/resources/audio/index.mjs'; -import { PluginOptions } from './index.js'; -import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; - -export type TranslationRequestBuilder = ( - req: GenerateRequest, - params: TranslationCreateParams -) => void; - -export const TRANSLATION_MODEL_INFO: ModelInfo = { - supports: { - media: true, - output: ['text', 'json'], - multiturn: false, - systemRole: false, - tools: false, - }, -}; - -export const TranslationConfigSchema = GenerationCommonConfigSchema.pick({ - temperature: true, -}).extend({ - response_format: z - .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) - .optional(), -}); - -function toTranslationRequest( - modelName: string, - request: GenerateRequest, - requestBuilder?: TranslationRequestBuilder -): TranslationCreateParams { - const message = new Message(request.messages[0]); - const media = message.media; - if (!media?.url) { - throw new Error('No media found in the request'); - } - const mediaBuffer = Buffer.from( - media.url.slice(media.url.indexOf(',') + 1), - 'base64' - ); - const mediaFile = new File([mediaBuffer], 'input', { - type: - media.contentType ?? - media.url.slice('data:'.length, media.url.indexOf(';')), - }); - const { - temperature, - version: modelVersion, - maxOutputTokens, - stopSequences, - topK, - topP, - ...restOfConfig - } = request.config ?? {}; - - let options: TranslationCreateParams = { - model: modelVersion ?? modelName, - file: mediaFile, - prompt: message.text, - temperature, - }; - if (requestBuilder) { - requestBuilder(request, options); - } else { - options = { - ...options, - ...restOfConfig, // passthrough rest of the config - }; - } - const outputFormat = request.output?.format as 'json' | 'text' | 'media'; - const customFormat = request.config?.response_format; - if (outputFormat && customFormat) { - if ( - outputFormat === 'json' && - customFormat !== 'json' && - customFormat !== 'verbose_json' - ) { - throw new Error( - `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` - ); - } - } - if (outputFormat === 'media') { - throw new Error(`Output format ${outputFormat} is not supported.`); - } - options.response_format = customFormat || outputFormat || 'text'; - for (const k in options) { - if (options[k] === undefined) { - delete options[k]; - } - } - return options; -} - -function translationToGenerateResponse( - result: TranslationCreateResponse | string -): GenerateResponseData { - return { - message: { - role: 'model', - content: [ - { - text: typeof result === 'string' ? result : result.text, - }, - ], - }, - finishReason: 'stop', - raw: result, - }; -} - -/** - * Method to define a new Genkit Model that is compatible with Open AI - * Translation API. - * - * These models are to be used to translate audio to text. - * - * @param params An object containing parameters for defining the OpenAI - * translation model. - * @param params.ai The Genkit AI instance. - * @param params.name The name of the model. - * @param params.client The OpenAI client instance. - * @param params.modelRef Optional reference to the model's configuration and - * custom options. - * - * @returns the created {@link ModelAction} - */ -export function defineCompatOpenAITranslationModel< - CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, ->(params: { - name: string; - client: OpenAI; - pluginOptions?: PluginOptions; - modelRef?: ModelReference; - requestBuilder?: TranslationRequestBuilder; -}) { - const { - name, - client: defaultClient, - pluginOptions, - modelRef, - requestBuilder, - } = params; - const modelName = toModelName(name, pluginOptions?.name); - const actionName = `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; - - return model( - { - name: actionName, - ...modelRef?.info, - configSchema: modelRef?.configSchema, - }, - async (request, { abortSignal }) => { - const params = toTranslationRequest(modelName, request, requestBuilder); - const client = maybeCreateRequestScopedOpenAIClient( - pluginOptions, - request, - defaultClient - ); - const result = await client.audio.translations.create(params, { - signal: abortSignal, - }); - return translationToGenerateResponse(result); - } - ); -} - -/** Translation ModelRef helper, with reasonable defaults for - * OpenAI-compatible providers */ -export function compatOaiTranslationModelRef< - CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, ->(params: { - name: string; - info?: ModelInfo; - configSchema?: CustomOptions; - config?: any; - namespace?: string; -}) { - const { - name, - info = TRANSLATION_MODEL_INFO, - configSchema, - config = undefined, - namespace, - } = params; - return modelRef({ - name, - configSchema: configSchema || (TranslationConfigSchema as any), - info, - config, - namespace, - }); -} diff --git a/js/testapps/compat-oai/src/index.ts b/js/testapps/compat-oai/src/index.ts index 506eebb611..3f67ec3385 100644 --- a/js/testapps/compat-oai/src/index.ts +++ b/js/testapps/compat-oai/src/index.ts @@ -382,7 +382,9 @@ ai.defineFlow('translate', async () => { const audioFile = fs.readFileSync('audio-korean.mp3'); const { text } = await ai.generate({ - model: openAI.model('whisper-1-translate'), + model: openAI.model('whisper-1', { + translate: true, + }), prompt: [ { media: { From 6689e38652a78a13e2ff90626eb876ab68b34918 Mon Sep 17 00:00:00 2001 From: 7hokerz Date: Wed, 25 Feb 2026 17:16:13 +0900 Subject: [PATCH 4/4] refactor: remain translate file and add test about whisper-specific --- js/plugins/compat-oai/src/audio.ts | 213 +------------- js/plugins/compat-oai/src/index.ts | 10 +- js/plugins/compat-oai/src/openai/index.ts | 13 +- js/plugins/compat-oai/src/openai/whisper.ts | 119 +++++++- js/plugins/compat-oai/src/translate.ts | 223 ++++++++++++++ .../tests/compat_oai_translate_test.ts | 277 ++++++++++++++++++ .../compat-oai/tests/openai_whisper_test.ts | 205 +++++++++++++ js/testapps/compat-oai/src/index.ts | 1 + 8 files changed, 835 insertions(+), 226 deletions(-) create mode 100644 js/plugins/compat-oai/src/translate.ts create mode 100644 js/plugins/compat-oai/tests/compat_oai_translate_test.ts create mode 100644 js/plugins/compat-oai/tests/openai_whisper_test.ts diff --git a/js/plugins/compat-oai/src/audio.ts b/js/plugins/compat-oai/src/audio.ts index 0a9dbbd25a..0a19989aaf 100644 --- a/js/plugins/compat-oai/src/audio.ts +++ b/js/plugins/compat-oai/src/audio.ts @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import type { GenerateRequest, GenerateResponseData, @@ -28,8 +29,6 @@ import type { SpeechCreateParams, Transcription, TranscriptionCreateParams, - TranslationCreateParams, - TranslationCreateResponse, } from 'openai/resources/audio/index.mjs'; import { PluginOptions } from './index.js'; import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; @@ -42,10 +41,6 @@ export type TranscriptionRequestBuilder = ( req: GenerateRequest, params: TranscriptionCreateParams ) => void; -export type TranslationRequestBuilder = ( - req: GenerateRequest, - params: TranslationCreateParams -) => void; export const TRANSCRIPTION_MODEL_INFO: ModelInfo = { supports: { @@ -67,16 +62,6 @@ export const SPEECH_MODEL_INFO: ModelInfo = { }, }; -export const WHISPER_MODER_INFO: ModelInfo = { - supports: { - media: true, - output: ['text', 'json'], - multiturn: false, - systemRole: false, - tools: false, - }, -}; - const ChunkingStrategySchema = z.object({ type: z.string(), prefix_padding_ms: z.number().int().optional(), @@ -108,19 +93,6 @@ export const SpeechConfigSchema = z.object({ .optional(), }); -export const WhisperConfigSchema = GenerationCommonConfigSchema.pick({ - temperature: true, -}).extend({ - /** When true, uses Translation API. Default: false **/ - translate: z.boolean().optional().default(false), - response_format: z - .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) - .optional(), - // transcription-only fields (ignored when translate=true) - language: z.string().optional(), - timestamp_granularities: z.array(z.enum(['word', 'segment'])).optional(), -}); - /** * Supported media formats for Audio generation */ @@ -449,186 +421,3 @@ export function compatOaiTranscriptionModelRef< namespace, }); } - -function toTranslationRequest( - modelName: string, - request: GenerateRequest, - requestBuilder?: TranslationRequestBuilder -): TranslationCreateParams { - const message = new Message(request.messages[0]); - const media = message.media; - if (!media?.url) { - throw new Error('No media found in the request'); - } - const mediaBuffer = Buffer.from( - media.url.slice(media.url.indexOf(',') + 1), - 'base64' - ); - const mediaFile = new File([mediaBuffer], 'input', { - type: - media.contentType ?? - media.url.slice('data:'.length, media.url.indexOf(';')), - }); - const { - temperature, - version: modelVersion, - maxOutputTokens, - stopSequences, - topK, - topP, - ...restOfConfig - } = request.config ?? {}; - - let options: TranslationCreateParams = { - model: modelVersion ?? modelName, - file: mediaFile, - prompt: message.text, - temperature, - }; - if (requestBuilder) { - requestBuilder(request, options); - } else { - options = { - ...options, - ...restOfConfig, // passthrough rest of the config - }; - } - const outputFormat = request.output?.format as 'json' | 'text' | 'media'; - const customFormat = request.config?.response_format; - if (outputFormat && customFormat) { - if ( - outputFormat === 'json' && - customFormat !== 'json' && - customFormat !== 'verbose_json' - ) { - throw new Error( - `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` - ); - } - } - if (outputFormat === 'media') { - throw new Error(`Output format ${outputFormat} is not supported.`); - } - options.response_format = customFormat || outputFormat || 'text'; - for (const k in options) { - if (options[k] === undefined) { - delete options[k]; - } - } - return options; -} - -function translationToGenerateResponse( - result: TranslationCreateResponse | string -): GenerateResponseData { - return { - message: { - role: 'model', - content: [ - { - text: typeof result === 'string' ? result : result.text, - }, - ], - }, - finishReason: 'stop', - raw: result, - }; -} - -/** - * Method to define a new Genkit Model that is compatible with Open AI - * Whisper API. - * - * These models are to be used to transcribe or translate audio to text. - * - * @param params An object containing parameters for defining the OpenAI - * whisper model. - * @param params.ai The Genkit AI instance. - * @param params.name The name of the model. - * @param params.client The OpenAI client instance. - * @param params.modelRef Optional reference to the model's configuration and - * custom options. - * - * @returns the created {@link ModelAction} - */ -export function defineCompatOpenAIWhisperModel(params: { - name: string; - client: OpenAI; - pluginOptions?: PluginOptions; - modelRef?: ModelReference; - requestBuilder?: TranscriptionRequestBuilder | TranslationRequestBuilder; -}) { - const { - name, - pluginOptions, - client: defaultClient, - modelRef, - requestBuilder, - } = params; - const modelName = toModelName(name, pluginOptions?.name); - const actionName = - modelRef?.name ?? `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; - - return model( - { - name: actionName, - ...modelRef?.info, - configSchema: modelRef?.configSchema, - }, - async (request, { abortSignal }) => { - const isTranslate = request.config?.translate === true; - const client = maybeCreateRequestScopedOpenAIClient( - pluginOptions, - request, - defaultClient - ); - - if (isTranslate) { - // Translation API - const params = toTranslationRequest(modelName, request, requestBuilder); - const result = await client.audio.translations.create(params, { - signal: abortSignal, - }); - return translationToGenerateResponse(result); - } else { - // Transcription API - const params = toSttRequest(modelName, request, requestBuilder); - const result = await client.audio.transcriptions.create( - { - ...params, - stream: false, - }, - { signal: abortSignal } - ); - return transcriptionToGenerateResponse(result); - } - } - ); -} - -/** Whisper ModelRef helper, with reasonable defaults for - * OpenAI-compatible providers */ -export function compatOaiWhisperModelRef< - CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, ->(params: { - name: string; - info?: ModelInfo; - configSchema?: CustomOptions; - config?: any; - namespace?: string; -}) { - const { - name, - info = WHISPER_MODER_INFO, - configSchema, - config = undefined, - namespace, - } = params; - return modelRef({ - name, - configSchema: configSchema || (WhisperConfigSchema as any), - info, - config, - namespace, - }); -} diff --git a/js/plugins/compat-oai/src/index.ts b/js/plugins/compat-oai/src/index.ts index 19198d3496..f474ebbf9d 100644 --- a/js/plugins/compat-oai/src/index.ts +++ b/js/plugins/compat-oai/src/index.ts @@ -24,16 +24,12 @@ import { toModelName } from './utils.js'; export { SpeechConfigSchema, TranscriptionConfigSchema, - WhisperConfigSchema, compatOaiSpeechModelRef, compatOaiTranscriptionModelRef, - compatOaiWhisperModelRef, defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, - defineCompatOpenAIWhisperModel, type SpeechRequestBuilder, type TranscriptionRequestBuilder, - type TranslationRequestBuilder, } from './audio.js'; export { defineCompatOpenAIEmbedder } from './embedder.js'; export { @@ -49,6 +45,12 @@ export { openAIModelRunner, type ModelRequestBuilder, } from './model.js'; +export { + TranslationConfigSchema, + compatOaiTranslationModelRef, + defineCompatOpenAITranslationModel, + type TranslationRequestBuilder, +} from './translate.js'; export interface PluginOptions extends Partial> { apiKey?: ClientOptions['apiKey'] | false; diff --git a/js/plugins/compat-oai/src/openai/index.ts b/js/plugins/compat-oai/src/openai/index.ts index 29efb71d9d..15b4ec7671 100644 --- a/js/plugins/compat-oai/src/openai/index.ts +++ b/js/plugins/compat-oai/src/openai/index.ts @@ -30,10 +30,8 @@ import OpenAI from 'openai'; import { defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel, - defineCompatOpenAIWhisperModel, SpeechConfigSchema, TranscriptionConfigSchema, - WhisperConfigSchema, } from '../audio.js'; import { defineCompatOpenAIEmbedder } from '../embedder.js'; import { @@ -58,7 +56,12 @@ import { } from './gpt.js'; import { openAITranscriptionModelRef, SUPPORTED_STT_MODELS } from './stt.js'; import { openAISpeechModelRef, SUPPORTED_TTS_MODELS } from './tts.js'; -import { openAIWhisperModelRef, SUPPORTED_WHISPER_MODELS } from './whisper.js'; +import { + defineOpenAIWhisperModel, + openAIWhisperModelRef, + SUPPORTED_WHISPER_MODELS, + WhisperConfigSchema, +} from './whisper.js'; export type OpenAIPluginOptions = Omit; @@ -93,7 +96,7 @@ function createResolver(pluginOptions: PluginOptions) { }); } else if (actionName.includes('whisper')) { const modelRef = openAIWhisperModelRef({ name: actionName }); - return defineCompatOpenAIWhisperModel({ + return defineOpenAIWhisperModel({ name: modelRef.name, client, pluginOptions, @@ -225,7 +228,7 @@ export function openAIPlugin(options?: OpenAIPluginOptions): GenkitPluginV2 { ); models.push( ...Object.values(SUPPORTED_WHISPER_MODELS).map((modelRef) => - defineCompatOpenAIWhisperModel({ + defineOpenAIWhisperModel({ name: modelRef.name, client, pluginOptions, diff --git a/js/plugins/compat-oai/src/openai/whisper.ts b/js/plugins/compat-oai/src/openai/whisper.ts index 5ea1952ab1..91d2ba67a6 100644 --- a/js/plugins/compat-oai/src/openai/whisper.ts +++ b/js/plugins/compat-oai/src/openai/whisper.ts @@ -15,11 +15,108 @@ * limitations under the License. */ -import { z } from 'genkit'; -import { ModelInfo } from 'genkit/model'; -import { compatOaiWhisperModelRef } from '../audio'; +import type { ModelReference } from 'genkit'; +import { modelRef, z } from 'genkit'; +import type { ModelAction, ModelInfo } from 'genkit/model'; +import { model } from 'genkit/plugin'; +import OpenAI from 'openai'; +import { + TranscriptionConfigSchema, + toSttRequest, + transcriptionToGenerateResponse, +} from '../audio.js'; +import type { PluginOptions } from '../index.js'; +import { + toTranslationRequest, + translationToGenerateResponse, +} from '../translate.js'; +import { maybeCreateRequestScopedOpenAIClient, toModelName } from '../utils.js'; -/** OpenAI whisper ModelRef helper, same as the OpenAI-compatible spec. */ +export const WHISPER_MODEL_INFO: ModelInfo = { + supports: { + media: true, + output: ['text', 'json'], + multiturn: false, + systemRole: false, + tools: false, + }, +}; + +/** + * Config schema for Whisper models. Extends the transcription config with + * a `translate` flag that switches between transcription and translation APIs. + */ +export const WhisperConfigSchema = TranscriptionConfigSchema.extend({ + /** When true, uses Translation API instead of Transcription. Default: false */ + translate: z.boolean().optional().default(false), +}); + +/** + * Method to define an OpenAI Whisper model that can perform both transcription and + * translation based on the `translate` config flag. + * + * @param params.ai The Genkit AI instance. + * @param params.name The name of the model. + * @param params.client The OpenAI client instance. + * @param params.modelRef Optional reference to the model's configuration and + * custom options. + * + * @returns the created {@link ModelAction} + */ +export function defineOpenAIWhisperModel< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + client: OpenAI; + modelRef?: ModelReference; + pluginOptions?: PluginOptions; +}): ModelAction { + const { name, client: defaultClient, pluginOptions, modelRef } = params; + const modelName = toModelName(name, pluginOptions?.name); + const actionName = + modelRef?.name ?? `${pluginOptions?.name ?? 'openai'}/${modelName}`; + + return model( + { + name: actionName, + ...modelRef?.info, + configSchema: modelRef?.configSchema, + }, + async (request, { abortSignal }) => { + const { translate, ...cleanConfig } = (request.config ?? {}) as Record< + string, + unknown + >; + const cleanRequest = { ...request, config: cleanConfig }; + const client = maybeCreateRequestScopedOpenAIClient( + pluginOptions, + request, + defaultClient + ); + + if (translate === true) { + const params = toTranslationRequest(modelName, cleanRequest); + const result = await client.audio.translations.create(params, { + signal: abortSignal, + }); + return translationToGenerateResponse(result); + } else { + const params = toSttRequest(modelName, cleanRequest); + // Explicitly setting stream to false ensures we use the non-streaming overload + const result = await client.audio.transcriptions.create( + { + ...params, + stream: false, + }, + { signal: abortSignal } + ); + return transcriptionToGenerateResponse(result); + } + } + ); +} + +/** OpenAI whisper ModelRef helper. */ export function openAIWhisperModelRef< CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, >(params: { @@ -28,7 +125,19 @@ export function openAIWhisperModelRef< configSchema?: CustomOptions; config?: any; }) { - return compatOaiWhisperModelRef({ ...params, namespace: 'openai' }); + const { + name, + info = WHISPER_MODEL_INFO, + configSchema, + config = undefined, + } = params; + return modelRef({ + name, + configSchema: configSchema || (WhisperConfigSchema as any), + info, + config, + namespace: 'openai', + }); } export const SUPPORTED_WHISPER_MODELS = { diff --git a/js/plugins/compat-oai/src/translate.ts b/js/plugins/compat-oai/src/translate.ts new file mode 100644 index 0000000000..2200b5e119 --- /dev/null +++ b/js/plugins/compat-oai/src/translate.ts @@ -0,0 +1,223 @@ +/** + * Copyright 2024 The Fire Company + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { + GenerateRequest, + GenerateResponseData, + ModelReference, +} from 'genkit'; +import { GenerationCommonConfigSchema, Message, modelRef, z } from 'genkit'; +import type { ModelAction, ModelInfo } from 'genkit/model'; +import { model } from 'genkit/plugin'; +import OpenAI from 'openai'; +import type { + TranslationCreateParams, + TranslationCreateResponse, +} from 'openai/resources/audio/index.mjs'; +import { PluginOptions } from './index.js'; +import { maybeCreateRequestScopedOpenAIClient, toModelName } from './utils.js'; + +export type TranslationRequestBuilder = ( + req: GenerateRequest, + params: TranslationCreateParams +) => void; + +export const TRANSLATION_MODEL_INFO: ModelInfo = { + supports: { + media: true, + output: ['text', 'json'], + multiturn: false, + systemRole: false, + tools: false, + }, +}; + +export const TranslationConfigSchema = GenerationCommonConfigSchema.pick({ + temperature: true, +}).extend({ + response_format: z + .enum(['json', 'text', 'srt', 'verbose_json', 'vtt']) + .optional(), +}); + +export function toTranslationRequest( + modelName: string, + request: GenerateRequest, + requestBuilder?: TranslationRequestBuilder +): TranslationCreateParams { + const message = new Message(request.messages[0]); + const media = message.media; + if (!media?.url) { + throw new Error('No media found in the request'); + } + const mediaBuffer = Buffer.from( + media.url.slice(media.url.indexOf(',') + 1), + 'base64' + ); + const mediaFile = new File([mediaBuffer], 'input', { + type: + media.contentType ?? + media.url.slice('data:'.length, media.url.indexOf(';')), + }); + const { + temperature, + version: modelVersion, + maxOutputTokens, + stopSequences, + topK, + topP, + ...restOfConfig + } = request.config ?? {}; + + let options: TranslationCreateParams = { + model: modelVersion ?? modelName, + file: mediaFile, + prompt: message.text, + temperature, + }; + if (requestBuilder) { + requestBuilder(request, options); + } else { + options = { + ...options, + ...restOfConfig, // passthrough rest of the config + }; + } + const outputFormat = request.output?.format as 'json' | 'text' | 'media'; + const customFormat = request.config?.response_format; + if (outputFormat && customFormat) { + if ( + outputFormat === 'json' && + customFormat !== 'json' && + customFormat !== 'verbose_json' + ) { + throw new Error( + `Custom response format ${customFormat} is not compatible with output format ${outputFormat}` + ); + } + } + if (outputFormat === 'media') { + throw new Error(`Output format ${outputFormat} is not supported.`); + } + options.response_format = customFormat || outputFormat || 'text'; + for (const k in options) { + if (options[k] === undefined) { + delete options[k]; + } + } + return options; +} + +export function translationToGenerateResponse( + result: TranslationCreateResponse | string +): GenerateResponseData { + return { + message: { + role: 'model', + content: [ + { + text: typeof result === 'string' ? result : result.text, + }, + ], + }, + finishReason: 'stop', + raw: result, + }; +} + +/** + * Method to define a new Genkit Model that is compatible with Open AI + * Translation API. + * + * These models are to be used to translate audio to text. + * + * @param params An object containing parameters for defining the OpenAI + * translation model. + * @param params.ai The Genkit AI instance. + * @param params.name The name of the model. + * @param params.client The OpenAI client instance. + * @param params.modelRef Optional reference to the model's configuration and + * custom options. + * + * @returns the created {@link ModelAction} + */ +export function defineCompatOpenAITranslationModel< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + client: OpenAI; + pluginOptions?: PluginOptions; + modelRef?: ModelReference; + requestBuilder?: TranslationRequestBuilder; +}) { + const { + name, + client: defaultClient, + pluginOptions, + modelRef, + requestBuilder, + } = params; + const modelName = toModelName(name, pluginOptions?.name); + const actionName = `${pluginOptions?.name ?? 'compat-oai'}/${modelName}`; + + return model( + { + name: actionName, + ...modelRef?.info, + configSchema: modelRef?.configSchema, + }, + async (request, { abortSignal }) => { + const params = toTranslationRequest(modelName, request, requestBuilder); + const client = maybeCreateRequestScopedOpenAIClient( + pluginOptions, + request, + defaultClient + ); + const result = await client.audio.translations.create(params, { + signal: abortSignal, + }); + return translationToGenerateResponse(result); + } + ); +} + +/** Translation ModelRef helper, with reasonable defaults for + * OpenAI-compatible providers */ +export function compatOaiTranslationModelRef< + CustomOptions extends z.ZodTypeAny = z.ZodTypeAny, +>(params: { + name: string; + info?: ModelInfo; + configSchema?: CustomOptions; + config?: any; + namespace?: string; +}) { + const { + name, + info = TRANSLATION_MODEL_INFO, + configSchema, + config = undefined, + namespace, + } = params; + return modelRef({ + name, + configSchema: configSchema || (TranslationConfigSchema as any), + info, + config, + namespace, + }); +} diff --git a/js/plugins/compat-oai/tests/compat_oai_translate_test.ts b/js/plugins/compat-oai/tests/compat_oai_translate_test.ts new file mode 100644 index 0000000000..371bbdc8c0 --- /dev/null +++ b/js/plugins/compat-oai/tests/compat_oai_translate_test.ts @@ -0,0 +1,277 @@ +/** + * Copyright 2024 The Fire Company + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it, jest } from '@jest/globals'; +import { GenerateRequest } from 'genkit'; +import OpenAI from 'openai'; +import { Translation } from 'openai/resources/audio/translations.mjs'; +import { + defineCompatOpenAITranslationModel, + toTranslationRequest, + translationToGenerateResponse, +} from '../src/translate'; + +jest.mock('genkit/model', () => { + const originalModule = + jest.requireActual('genkit/model'); + return { + ...originalModule, + defineModel: jest.fn((_, runner) => { + return runner; + }), + }; +}); + +describe('toTranslationRequest', () => { + it('should create translation request from base64 audio', () => { + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + url: 'data:audio/wav;base64,aGVsbG8=', + }, + }, + ], + }, + ], + output: { format: 'text' }, + } as GenerateRequest; + + const actualOutput = toTranslationRequest('whisper-1', request); + expect(actualOutput).toStrictEqual({ + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'text', + }); + }); + + it('should allow verbose_json when output.format is json', () => { + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + url: 'data:audio/wav;base64,aGVsbG8=', + }, + }, + ], + }, + ], + output: { format: 'json' }, + config: { response_format: 'verbose_json' }, + } as GenerateRequest; + + const actualOutput = toTranslationRequest('whisper-1', request); + expect(actualOutput).toStrictEqual({ + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'verbose_json', + }); + }); + + it('should throw error when media.url is missing', () => { + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + }, + }, + ], + }, + ], + output: { format: 'text' }, + } as GenerateRequest; + + expect(() => toTranslationRequest('whisper-1', request)).toThrowError( + 'No media found in the request' + ); + }); + + it('should throw error when output.format is json but custom format is incompatible', () => { + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + url: 'data:audio/wav;base64,aGVsbG8=', + }, + }, + ], + }, + ], + output: { format: 'json' }, + config: { response_format: 'srt' }, + } as GenerateRequest; + + expect(() => toTranslationRequest('whisper-1', request)).toThrowError( + 'Custom response format srt is not compatible with output format json' + ); + }); + + it('should throw error when output.format is media', () => { + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + url: 'data:audio/wav;base64,aGVsbG8=', + }, + }, + ], + }, + ], + output: { format: 'media' }, + } as GenerateRequest; + + expect(() => toTranslationRequest('whisper-1', request)).toThrow( + 'Output format media is not supported.' + ); + }); + + it('should run with requestBuilder', () => { + const requestBuilder = jest.fn((_, params) => { + (params as any).foo = 'bar'; + }); + + const request = { + messages: [ + { + role: 'user', + content: [ + { + media: { + contentType: 'audio/wav', + url: 'data:audio/wav;base64,aGVsbG8=', + }, + }, + ], + }, + ], + output: { format: 'text' }, + } as GenerateRequest; + + const actualOutput = toTranslationRequest( + 'whisper-1', + request, + requestBuilder + ); + + expect(requestBuilder).toHaveBeenCalledTimes(1); + expect(actualOutput).toHaveProperty('foo', 'bar'); + }); +}); + +describe('translationToGenerateResponse', () => { + it('should transform translation result correctly when result is Translation object', () => { + const result: Translation = { + text: 'Hello', + }; + + const actualOutput = translationToGenerateResponse(result); + expect(actualOutput).toStrictEqual({ + message: { + role: 'model', + content: [{ text: 'Hello' }], + }, + finishReason: 'stop', + raw: result, + }); + }); + + it('should transform translation result correctly when result is string', () => { + const result = 'Hello'; + + const actualOutput = translationToGenerateResponse(result); + expect(actualOutput).toStrictEqual({ + message: { + role: 'model', + content: [{ text: 'Hello' }], + }, + finishReason: 'stop', + raw: result, + }); + }); +}); + +describe('defineCompatOpenAITranslationModel runner', () => { + it('should correctly run Translation requests', async () => { + const result: Translation = { + text: 'Hello', + }; + + const openaiClient = { + audio: { + translations: { + create: jest.fn(async () => result), + }, + }, + }; + const abortSignal = jest.fn(); + const runner = defineCompatOpenAITranslationModel({ + name: 'whisper-1', + client: openaiClient as unknown as OpenAI, + }); + await runner( + { + messages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:audio/wav;base64,aGVsbG8=', + contentType: 'audio/wav', + }, + }, + ], + }, + ], + }, + { + abortSignal: abortSignal as unknown as AbortSignal, + } + ); + expect(openaiClient.audio.translations.create).toHaveBeenCalledWith( + { + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'text', + }, + { signal: abortSignal } + ); + }); +}); diff --git a/js/plugins/compat-oai/tests/openai_whisper_test.ts b/js/plugins/compat-oai/tests/openai_whisper_test.ts new file mode 100644 index 0000000000..ff727c13f7 --- /dev/null +++ b/js/plugins/compat-oai/tests/openai_whisper_test.ts @@ -0,0 +1,205 @@ +/** + * Copyright 2024 The Fire Company + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it, jest } from '@jest/globals'; +import OpenAI from 'openai'; +import { Transcription } from 'openai/resources/audio/transcriptions.mjs'; +import { Translation } from 'openai/resources/audio/translations.mjs'; +import { defineOpenAIWhisperModel } from '../src/openai/whisper'; + +jest.mock('genkit/model', () => { + const originalModule = + jest.requireActual('genkit/model'); + return { + ...originalModule, + defineModel: jest.fn((_, runner) => { + return runner; + }), + }; +}); + +describe('defineOpenAIWhisperModel runner — transcription (default)', () => { + it('should call transcriptions.create when translate is not set', async () => { + const result: Transcription = { + text: 'Hello world', + }; + + const openaiClient = { + audio: { + transcriptions: { + create: jest.fn(async () => result), + }, + translations: { + create: jest.fn(async () => ({ text: 'should not be called' })), + }, + }, + }; + const abortSignal = jest.fn(); + + const runner = defineOpenAIWhisperModel({ + name: 'whisper-1', + client: openaiClient as unknown as OpenAI, + }); + + await runner( + { + messages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:audio/wav;base64,aGVsbG8=', + contentType: 'audio/wav', + }, + }, + ], + }, + ], + }, + { + abortSignal: abortSignal as unknown as AbortSignal, + } + ); + + expect(openaiClient.audio.transcriptions.create).toHaveBeenCalledWith( + { + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'text', + stream: false, + }, + { signal: abortSignal } + ); + expect(openaiClient.audio.translations.create).not.toHaveBeenCalled(); + }); + + it('should call transcriptions.create when translate is explicitly false', async () => { + const result: Transcription = { + text: 'transcribed text', + }; + + const openaiClient = { + audio: { + transcriptions: { + create: jest.fn(async () => result), + }, + translations: { + create: jest.fn(async () => ({ text: 'should not be called' })), + }, + }, + }; + const abortSignal = jest.fn(); + + const runner = defineOpenAIWhisperModel({ + name: 'whisper-1', + client: openaiClient as unknown as OpenAI, + }); + + await runner( + { + messages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:audio/wav;base64,aGVsbG8=', + contentType: 'audio/wav', + }, + }, + ], + }, + ], + config: { translate: false }, + }, + { + abortSignal: abortSignal as unknown as AbortSignal, + } + ); + expect(openaiClient.audio.transcriptions.create).toHaveBeenCalledWith( + { + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'text', + stream: false, + }, + { signal: abortSignal } + ); + expect(openaiClient.audio.translations.create).not.toHaveBeenCalled(); + }); +}); + +describe('defineOpenAIWhisperModel runner — translation (translate: true)', () => { + it('should call translations.create when translate is true', async () => { + const result: Translation = { + text: 'Hello in English', + }; + + const openaiClient = { + audio: { + transcriptions: { + create: jest.fn(async () => ({ text: 'should not be called' })), + }, + translations: { + create: jest.fn(async () => result), + }, + }, + }; + const abortSignal = jest.fn(); + + const runner = defineOpenAIWhisperModel({ + name: 'whisper-1', + client: openaiClient as unknown as OpenAI, + }); + + await runner( + { + messages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:audio/wav;base64,aGVsbG8=', + contentType: 'audio/wav', + }, + }, + ], + }, + ], + config: { translate: true }, + }, + { + abortSignal: abortSignal as unknown as AbortSignal, + } + ); + + expect(openaiClient.audio.translations.create).toHaveBeenCalledWith( + { + model: 'whisper-1', + file: expect.any(File), + prompt: '', + response_format: 'text', + }, + { signal: abortSignal } + ); + expect(openaiClient.audio.transcriptions.create).not.toHaveBeenCalled(); + }); +}); diff --git a/js/testapps/compat-oai/src/index.ts b/js/testapps/compat-oai/src/index.ts index 3f67ec3385..6f195838f2 100644 --- a/js/testapps/compat-oai/src/index.ts +++ b/js/testapps/compat-oai/src/index.ts @@ -384,6 +384,7 @@ ai.defineFlow('translate', async () => { const { text } = await ai.generate({ model: openAI.model('whisper-1', { translate: true, + temperature: 0.5, }), prompt: [ {