feat(ai): ImageConfig and FinishReasons

.changeset/pb-add-image-config.md

@@ -0,0 +1,6 @@
+---
+"@firebase/ai": minor
+"firebase": minor
+---
+
+Added support for `ImageConfig` (aspect ratio and size) and expanded `FinishReason` values to achieve parity with the iOS SDK.

common/api-review/ai.api.md

@@ -358,6 +358,16 @@ export const FinishReason: {
     readonly PROHIBITED_CONTENT: "PROHIBITED_CONTENT";
     readonly SPII: "SPII";
     readonly MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL";
+    readonly IMAGE_SAFETY: "IMAGE_SAFETY";
+    readonly IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT";
+    readonly IMAGE_OTHER: "IMAGE_OTHER";
+    readonly NO_IMAGE: "NO_IMAGE";
+    readonly IMAGE_RECITATION: "IMAGE_RECITATION";
+    readonly LANGUAGE: "LANGUAGE";
+    readonly UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL";
+    readonly TOO_MANY_TOOL_CALLS: "TOO_MANY_TOOL_CALLS";
+    readonly MISSING_THOUGHT_SIGNATURE: "MISSING_THOUGHT_SIGNATURE";
+    readonly MALFORMED_RESPONSE: "MALFORMED_RESPONSE";
 };
 
 // @public
@@ -516,6 +526,7 @@ export interface GenerationConfig {
     candidateCount?: number;
     // (undocumented)
     frequencyPenalty?: number;
+    imageConfig?: ImageConfig;
     // (undocumented)
     maxOutputTokens?: number;
     // (undocumented)
@@ -744,6 +755,44 @@ export interface HybridParams {
     onDeviceParams?: OnDeviceParams;
 }
 
+// @public
+export interface ImageConfig {
+    aspectRatio?: ImageConfigAspectRatio;
+    imageSize?: ImageConfigImageSize;
+}
+
+// @public
+export const ImageConfigAspectRatio: {
+    readonly SQUARE_1x1: "1:1";
+    readonly PORTRAIT_9x16: "9:16";
+    readonly LANDSCAPE_16x9: "16:9";
+    readonly PORTRAIT_3x4: "3:4";
+    readonly LANDSCAPE_4x3: "4:3";
+    readonly PORTRAIT_2x3: "2:3";
+    readonly LANDSCAPE_3x2: "3:2";
+    readonly PORTRAIT_4x5: "4:5";
+    readonly LANDSCAPE_5x4: "5:4";
+    readonly PORTRAIT_1x4: "1:4";
+    readonly LANDSCAPE_4x1: "4:1";
+    readonly PORTRAIT_1x8: "1:8";
+    readonly LANDSCAPE_8x1: "8:1";
+    readonly ULTRAWIDE_21x9: "21:9";
+};
+
+// @public
+export type ImageConfigAspectRatio = (typeof ImageConfigAspectRatio)[keyof typeof ImageConfigAspectRatio];
+
+// @public
+export const ImageConfigImageSize: {
+    readonly SIZE_512: "512";
+    readonly SIZE_1K: "1K";
+    readonly SIZE_2K: "2K";
+    readonly SIZE_4K: "4K";
+};
+
+// @public
+export type ImageConfigImageSize = (typeof ImageConfigImageSize)[keyof typeof ImageConfigImageSize];
+
 // @public @deprecated
 export const ImagenAspectRatio: {
     readonly SQUARE: "1:1";

docs-devsite/_toc.yaml

@@ -106,6 +106,8 @@ toc:
     path: /docs/reference/js/ai.groundingsupport.md
   - title: HybridParams
     path: /docs/reference/js/ai.hybridparams.md
+  - title: ImageConfig
+    path: /docs/reference/js/ai.imageconfig.md
   - title: ImagenGCSImage
     path: /docs/reference/js/ai.imagengcsimage.md
   - title: ImagenGenerationConfig

docs-devsite/ai.generationconfig.md

@@ -24,6 +24,7 @@ export interface GenerationConfig
 |  --- | --- | --- |
 |  [candidateCount](./ai.generationconfig.md#generationconfigcandidatecount) | number |  |
 |  [frequencyPenalty](./ai.generationconfig.md#generationconfigfrequencypenalty) | number |  |
+|  [imageConfig](./ai.generationconfig.md#generationconfigimageconfig) | [ImageConfig](./ai.imageconfig.md#imageconfig_interface) | Configuration options for generating images with Gemini models. |
 |  [maxOutputTokens](./ai.generationconfig.md#generationconfigmaxoutputtokens) | number |  |
 |  [presencePenalty](./ai.generationconfig.md#generationconfigpresencepenalty) | number |  |
 |  [responseJsonSchema](./ai.generationconfig.md#generationconfigresponsejsonschema) | { \[key: string\]: unknown; } | Output schema of the generated response. This is an alternative to <code>responseSchema</code> that accepts \[JSON Schema\](https://json-schema.org/).<!-- -->If set, <code>responseSchema</code> must be omitted, but <code>responseMimeType</code> is required and must be set to <code>application/json</code>. |
@@ -52,6 +53,16 @@ candidateCount?: number;
 frequencyPenalty?: number;
 ```
 
+## GenerationConfig.imageConfig
+
+Configuration options for generating images with Gemini models.
+
+<b>Signature:</b>
+
+```typescript
+imageConfig?: ImageConfig;
+```
+
 ## GenerationConfig.maxOutputTokens
 
 <b>Signature:</b>

docs-devsite/ai.imageconfig.md

@@ -0,0 +1,46 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# ImageConfig interface
+Configuration options for generating images with Gemini models.
+
+<b>Signature:</b>
+
+```typescript
+export interface ImageConfig 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [aspectRatio](./ai.imageconfig.md#imageconfigaspectratio) | [ImageConfigAspectRatio](./ai.md#imageconfigaspectratio) | The aspect ratio of generated images. |
+|  [imageSize](./ai.imageconfig.md#imageconfigimagesize) | [ImageConfigImageSize](./ai.md#imageconfigimagesize) | The size of the generated images. |
+
+## ImageConfig.aspectRatio
+
+The aspect ratio of generated images.
+
+<b>Signature:</b>
+
+```typescript
+aspectRatio?: ImageConfigAspectRatio;
+```
+
+## ImageConfig.imageSize
+
+The size of the generated images.
+
+<b>Signature:</b>
+
+```typescript
+imageSize?: ImageConfigImageSize;
+```

docs-devsite/ai.md

@@ -99,6 +99,7 @@ The Firebase AI Web SDK.
 |  [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.<!-- -->Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)<!-- -->).<!-- -->Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). |
 |  [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. |
 |  [HybridParams](./ai.hybridparams.md#hybridparams_interface) | <b><i>(Public Preview)</i></b> Configures hybrid inference. |
+|  [ImageConfig](./ai.imageconfig.md#imageconfig_interface) | Configuration options for generating images with Gemini models. |
 |  [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.<!-- -->This feature is not available yet. |
 |  [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | Configuration options for generating images with Imagen.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. |
 |  [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | The response from a request to generate images with Imagen. |
@@ -166,6 +167,8 @@ The Firebase AI Web SDK.
 |  [HarmCategory](./ai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. |
 |  [HarmProbability](./ai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. |
 |  [HarmSeverity](./ai.md#harmseverity) | Harm severity levels. |
+|  [ImageConfigAspectRatio](./ai.md#imageconfigaspectratio) | Aspect ratios for generated images. |
+|  [ImageConfigImageSize](./ai.md#imageconfigimagesize) | Sizes for generated images. For example, '1K' is 1024px, '2K' is 2048px, and '4K' is 4096px. |
 |  [ImagenAspectRatio](./ai.md#imagenaspectratio) | Aspect ratios for Imagen images.<!-- -->To specify an aspect ratio for generated images, set the <code>aspectRatio</code> property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface)<!-- -->.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. |
 |  [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | A filter level controlling whether generation of images containing people or faces is allowed.<!-- -->See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a> documentation for more details. |
 |  [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | A filter level controlling how aggressively to filter sensitive content.<!-- -->Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, <code>violence</code>, <code>sexual</code>, <code>derogatory</code>, and <code>toxic</code>). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. |
@@ -195,6 +198,8 @@ The Firebase AI Web SDK.
 |  [HarmCategory](./ai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. |
 |  [HarmProbability](./ai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. |
 |  [HarmSeverity](./ai.md#harmseverity) | Harm severity levels. |
+|  [ImageConfigAspectRatio](./ai.md#imageconfigaspectratio) | Aspect ratios for generated images. |
+|  [ImageConfigImageSize](./ai.md#imageconfigimagesize) | Sizes for generated images. |
 |  [ImagenAspectRatio](./ai.md#imagenaspectratio) | Aspect ratios for Imagen images.<!-- -->To specify an aspect ratio for generated images, set the <code>aspectRatio</code> property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface)<!-- -->.<!-- -->See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. |
 |  [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | A filter level controlling whether generation of images containing people or faces is allowed.<!-- -->See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a> documentation for more details. |
 |  [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | A filter level controlling how aggressively to filter sensitive content.<!-- -->Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, <code>violence</code>, <code>sexual</code>, <code>derogatory</code>, and <code>toxic</code>). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. |
@@ -546,6 +551,16 @@ FinishReason: {
     readonly PROHIBITED_CONTENT: "PROHIBITED_CONTENT";
     readonly SPII: "SPII";
     readonly MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL";
+    readonly IMAGE_SAFETY: "IMAGE_SAFETY";
+    readonly IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT";
+    readonly IMAGE_OTHER: "IMAGE_OTHER";
+    readonly NO_IMAGE: "NO_IMAGE";
+    readonly IMAGE_RECITATION: "IMAGE_RECITATION";
+    readonly LANGUAGE: "LANGUAGE";
+    readonly UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL";
+    readonly TOO_MANY_TOOL_CALLS: "TOO_MANY_TOOL_CALLS";
+    readonly MISSING_THOUGHT_SIGNATURE: "MISSING_THOUGHT_SIGNATURE";
+    readonly MALFORMED_RESPONSE: "MALFORMED_RESPONSE";
 }
 ```
 
@@ -637,6 +652,46 @@ HarmSeverity: {
 }
 ```
 
+## ImageConfigAspectRatio
+
+Aspect ratios for generated images.
+
+<b>Signature:</b>
+
+```typescript
+ImageConfigAspectRatio: {
+    readonly SQUARE_1x1: "1:1";
+    readonly PORTRAIT_9x16: "9:16";
+    readonly LANDSCAPE_16x9: "16:9";
+    readonly PORTRAIT_3x4: "3:4";
+    readonly LANDSCAPE_4x3: "4:3";
+    readonly PORTRAIT_2x3: "2:3";
+    readonly LANDSCAPE_3x2: "3:2";
+    readonly PORTRAIT_4x5: "4:5";
+    readonly LANDSCAPE_5x4: "5:4";
+    readonly PORTRAIT_1x4: "1:4";
+    readonly LANDSCAPE_4x1: "4:1";
+    readonly PORTRAIT_1x8: "1:8";
+    readonly LANDSCAPE_8x1: "8:1";
+    readonly ULTRAWIDE_21x9: "21:9";
+}
+```
+
+## ImageConfigImageSize
+
+Sizes for generated images. For example, '1K' is 1024px, '2K' is 2048px, and '4K' is 4096px.
+
+<b>Signature:</b>
+
+```typescript
+ImageConfigImageSize: {
+    readonly SIZE_512: "512";
+    readonly SIZE_1K: "1K";
+    readonly SIZE_2K: "2K";
+    readonly SIZE_4K: "4K";
+}
+```
+
 ## ImagenAspectRatio
 
 > Warning: This API is now obsolete.
@@ -980,6 +1035,26 @@ Harm severity levels.
 export type HarmSeverity = (typeof HarmSeverity)[keyof typeof HarmSeverity];
 ```
 
+## ImageConfigAspectRatio
+
+Aspect ratios for generated images.
+
+<b>Signature:</b>
+
+```typescript
+export type ImageConfigAspectRatio = (typeof ImageConfigAspectRatio)[keyof typeof ImageConfigAspectRatio];
+```
+
+## ImageConfigImageSize
+
+Sizes for generated images.
+
+<b>Signature:</b>
+
+```typescript
+export type ImageConfigImageSize = (typeof ImageConfigImageSize)[keyof typeof ImageConfigImageSize];
+```
+
 ## ImagenAspectRatio
 
 > Warning: This API is now obsolete.

packages/ai/src/models/generative-model.test.ts

@@ -22,7 +22,9 @@ import {
   InferenceMode,
   AIErrorCode,
   ChromeAdapter,
-  ThinkingLevel
+  ThinkingLevel,
+  ImageConfigAspectRatio,
+  ImageConfigImageSize
 } from '../public-types';
 import * as request from '../requests/request';
 import { SinonStub, match, restore, stub } from 'sinon';
@@ -322,6 +324,22 @@ describe('GenerativeModel', () => {
     });
     restore();
   });
+  it('passes imageConfig through to ChatSession', () => {
+    const genModel = new GenerativeModel(fakeAI, {
+      model: 'my-model',
+      generationConfig: {
+        imageConfig: {
+          aspectRatio: ImageConfigAspectRatio.SQUARE_1x1,
+          imageSize: ImageConfigImageSize.SIZE_512
+        }
+      }
+    });
+    const chatSession = genModel.startChat();
+    expect(chatSession.params?.generationConfig?.imageConfig).to.deep.equal({
+      aspectRatio: '1:1',
+      imageSize: '512'
+    });
+  });
   it('overrides base model params with startChatParams', () => {
     const genModel = new GenerativeModel(fakeAI, {
       model: 'my-model',

packages/ai/src/requests/response-helpers.ts

@@ -229,7 +229,24 @@ export function getInlineDataParts(
   }
 }
 
-const badFinishReasons = [FinishReason.RECITATION, FinishReason.SAFETY];
+const badFinishReasons = [
+  FinishReason.RECITATION,
+  FinishReason.SAFETY,
+  FinishReason.BLOCKLIST,
+  FinishReason.PROHIBITED_CONTENT,
+  FinishReason.SPII,
+  FinishReason.MALFORMED_FUNCTION_CALL,
+  FinishReason.IMAGE_SAFETY,
+  FinishReason.IMAGE_PROHIBITED_CONTENT,
+  FinishReason.IMAGE_OTHER,
+  FinishReason.NO_IMAGE,
+  FinishReason.IMAGE_RECITATION,
+  FinishReason.LANGUAGE,
+  FinishReason.UNEXPECTED_TOOL_CALL,
+  FinishReason.TOO_MANY_TOOL_CALLS,
+  FinishReason.MISSING_THOUGHT_SIGNATURE,
+  FinishReason.MALFORMED_RESPONSE
+];
 
 function hadBadFinishReason(candidate: GenerateContentCandidate): boolean {
   return (