diff --git a/.changeset/purple-chairs-wait.md b/.changeset/purple-chairs-wait.md new file mode 100644 index 00000000000..e29595537cd --- /dev/null +++ b/.changeset/purple-chairs-wait.md @@ -0,0 +1,6 @@ +--- +'@firebase/ai': minor +'firebase': minor +--- + +Add hybrid inference options to the Firebase AI SDK. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 199b97b10a9..e0eac35996a 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -125,7 +125,7 @@ export class BooleanSchema extends Schema { // @public export class ChatSession { - constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); + constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); getHistory(): Promise; // (undocumented) model: string; @@ -137,6 +137,15 @@ export class ChatSession { sendMessageStream(request: string | Array): Promise; } +// @public +export interface ChromeAdapter { + // @internal (undocumented) + countTokens(request: CountTokensRequest): Promise; + generateContent(request: GenerateContentRequest): Promise; + generateContentStream(request: GenerateContentRequest): Promise; + isAvailable(request: GenerateContentRequest): Promise; +} + // @public export interface Citation { // (undocumented) @@ -416,7 +425,7 @@ export interface GenerativeContentBlob { // @public export class GenerativeModel extends AIModel { - constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); + constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined); countTokens(request: CountTokensRequest | string | Array): Promise; generateContent(request: GenerateContentRequest | string | Array): Promise; generateContentStream(request: GenerateContentRequest | string | Array): Promise; @@ -439,7 +448,7 @@ export class GenerativeModel extends AIModel { export function getAI(app?: FirebaseApp, options?: AIOptions): AI; // @public -export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; // @beta export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; @@ -588,6 +597,13 @@ export const HarmSeverity: { // @public export type HarmSeverity = (typeof HarmSeverity)[keyof typeof HarmSeverity]; +// @public +export interface HybridParams { + inCloudParams?: ModelParams; + mode: InferenceMode; + onDeviceParams?: OnDeviceParams; +} + // @beta export const ImagenAspectRatio: { readonly SQUARE: "1:1"; @@ -600,7 +616,7 @@ export const ImagenAspectRatio: { // @beta export type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio]; -// @public +// @beta export interface ImagenGCSImage { gcsURI: string; mimeType: string; @@ -681,6 +697,16 @@ export interface ImagenSafetySettings { safetyFilterLevel?: ImagenSafetyFilterLevel; } +// @public +export const InferenceMode: { + readonly PREFER_ON_DEVICE: "prefer_on_device"; + readonly ONLY_ON_DEVICE: "only_on_device"; + readonly ONLY_IN_CLOUD: "only_in_cloud"; +}; + +// @public +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; + // @public export interface InlineDataPart { // (undocumented) @@ -699,6 +725,63 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export interface LanguageModelCreateCoreOptions { + // (undocumented) + expectedInputs?: LanguageModelExpected[]; + // (undocumented) + temperature?: number; + // (undocumented) + topK?: number; +} + +// @public +export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { + // (undocumented) + initialPrompts?: LanguageModelMessage[]; + // (undocumented) + signal?: AbortSignal; +} + +// @public +export interface LanguageModelExpected { + // (undocumented) + languages?: string[]; + // (undocumented) + type: LanguageModelMessageType; +} + +// @public +export interface LanguageModelMessage { + // (undocumented) + content: LanguageModelMessageContent[]; + // (undocumented) + role: LanguageModelMessageRole; +} + +// @public +export interface LanguageModelMessageContent { + // (undocumented) + type: LanguageModelMessageType; + // (undocumented) + value: LanguageModelMessageContentValue; +} + +// @public +export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; + +// @public +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; + +// @public +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; + +// @public +export interface LanguageModelPromptOptions { + // (undocumented) + responseConstraint?: object; +} + // @public export const Modality: { readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED"; @@ -757,6 +840,14 @@ export interface ObjectSchemaRequest extends SchemaRequest { type: 'object'; } +// @public +export interface OnDeviceParams { + // (undocumented) + createOptions?: LanguageModelCreateOptions; + // (undocumented) + promptOptions?: LanguageModelPromptOptions; +} + // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 6d548ffd8d6..da7c2500894 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -24,6 +24,8 @@ toc: path: /docs/reference/js/ai.booleanschema.md - title: ChatSession path: /docs/reference/js/ai.chatsession.md + - title: ChromeAdapter + path: /docs/reference/js/ai.chromeadapter.md - title: Citation path: /docs/reference/js/ai.citation.md - title: CitationMetadata @@ -88,6 +90,8 @@ toc: path: /docs/reference/js/ai.groundingmetadata.md - title: GroundingSupport path: /docs/reference/js/ai.groundingsupport.md + - title: HybridParams + path: /docs/reference/js/ai.hybridparams.md - title: ImagenGCSImage path: /docs/reference/js/ai.imagengcsimage.md - title: ImagenGenerationConfig @@ -108,6 +112,18 @@ toc: path: /docs/reference/js/ai.inlinedatapart.md - title: IntegerSchema path: /docs/reference/js/ai.integerschema.md + - title: LanguageModelCreateCoreOptions + path: /docs/reference/js/ai.languagemodelcreatecoreoptions.md + - title: LanguageModelCreateOptions + path: /docs/reference/js/ai.languagemodelcreateoptions.md + - title: LanguageModelExpected + path: /docs/reference/js/ai.languagemodelexpected.md + - title: LanguageModelMessage + path: /docs/reference/js/ai.languagemodelmessage.md + - title: LanguageModelMessageContent + path: /docs/reference/js/ai.languagemodelmessagecontent.md + - title: LanguageModelPromptOptions + path: /docs/reference/js/ai.languagemodelpromptoptions.md - title: ModalityTokenCount path: /docs/reference/js/ai.modalitytokencount.md - title: ModelParams @@ -118,6 +134,8 @@ toc: path: /docs/reference/js/ai.objectschema.md - title: ObjectSchemaRequest path: /docs/reference/js/ai.objectschemarequest.md + - title: OnDeviceParams + path: /docs/reference/js/ai.ondeviceparams.md - title: PromptFeedback path: /docs/reference/js/ai.promptfeedback.md - title: RequestOptions diff --git a/docs-devsite/ai.chatsession.md b/docs-devsite/ai.chatsession.md index 1d6e403b6a8..4e4358898a5 100644 --- a/docs-devsite/ai.chatsession.md +++ b/docs-devsite/ai.chatsession.md @@ -22,7 +22,7 @@ export declare class ChatSession | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(apiSettings, model, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) | | Constructs a new instance of the ChatSession class | +| [(constructor)(apiSettings, model, chromeAdapter, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) | | Constructs a new instance of the ChatSession class | ## Properties @@ -47,7 +47,7 @@ Constructs a new instance of the `ChatSession` class Signature: ```typescript -constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); +constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); ``` #### Parameters @@ -56,6 +56,7 @@ constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | | --- | --- | --- | | apiSettings | ApiSettings | | | model | string | | +| chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined | | | params | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) \| undefined | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) \| undefined | | diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md new file mode 100644 index 00000000000..f497312c609 --- /dev/null +++ b/docs-devsite/ai.chromeadapter.md @@ -0,0 +1,94 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ChromeAdapter interface +(EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible. + +These methods should not be called directly by the user. + +Signature: + +```typescript +export interface ChromeAdapter +``` + +## Methods + +| Method | Description | +| --- | --- | +| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content using on-device inference.

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.

| +| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates a content stream using on-device inference.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.

| +| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if the on-device model is capable of handling a given request. | + +## ChromeAdapter.generateContent() + +Generates content using on-device inference. + +

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.

+ +Signature: + +```typescript +generateContent(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | + +Returns: + +Promise<Response> + +## ChromeAdapter.generateContentStream() + +Generates a content stream using on-device inference. + +

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.

+ +Signature: + +```typescript +generateContentStream(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | + +Returns: + +Promise<Response> + +## ChromeAdapter.isAvailable() + +Checks if the on-device model is capable of handling a given request. + +Signature: + +```typescript +isAvailable(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | A potential request to be passed to the model. | + +Returns: + +Promise<boolean> + diff --git a/docs-devsite/ai.generativemodel.md b/docs-devsite/ai.generativemodel.md index d91cf80e881..323fcfe9d76 100644 --- a/docs-devsite/ai.generativemodel.md +++ b/docs-devsite/ai.generativemodel.md @@ -23,7 +23,7 @@ export declare class GenerativeModel extends AIModel | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(ai, modelParams, requestOptions)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | +| [(constructor)(ai, modelParams, requestOptions, chromeAdapter)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | ## Properties @@ -52,7 +52,7 @@ Constructs a new instance of the `GenerativeModel` class Signature: ```typescript -constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); +constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined); ``` #### Parameters @@ -62,6 +62,7 @@ constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); | ai | [AI](./ai.ai.md#ai_interface) | | | modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | +| chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined | | ## GenerativeModel.generationConfig diff --git a/docs-devsite/ai.hybridparams.md b/docs-devsite/ai.hybridparams.md new file mode 100644 index 00000000000..baf568217d3 --- /dev/null +++ b/docs-devsite/ai.hybridparams.md @@ -0,0 +1,57 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# HybridParams interface +(EXPERIMENTAL) Configures hybrid inference. + +Signature: + +```typescript +export interface HybridParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [inCloudParams](./ai.hybridparams.md#hybridparamsincloudparams) | [ModelParams](./ai.modelparams.md#modelparams_interface) | Optional. Specifies advanced params for in-cloud inference. | +| [mode](./ai.hybridparams.md#hybridparamsmode) | [InferenceMode](./ai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. | +| [onDeviceParams](./ai.hybridparams.md#hybridparamsondeviceparams) | [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Optional. Specifies advanced params for on-device inference. | + +## HybridParams.inCloudParams + +Optional. Specifies advanced params for in-cloud inference. + +Signature: + +```typescript +inCloudParams?: ModelParams; +``` + +## HybridParams.mode + +Specifies on-device or in-cloud inference. Defaults to prefer on-device. + +Signature: + +```typescript +mode: InferenceMode; +``` + +## HybridParams.onDeviceParams + +Optional. Specifies advanced params for on-device inference. + +Signature: + +```typescript +onDeviceParams?: OnDeviceParams; +``` diff --git a/docs-devsite/ai.imagengcsimage.md b/docs-devsite/ai.imagengcsimage.md index cd11d8ee354..ec51c714e0f 100644 --- a/docs-devsite/ai.imagengcsimage.md +++ b/docs-devsite/ai.imagengcsimage.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. This feature is not available yet. @@ -24,11 +27,14 @@ export interface ImagenGCSImage | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./ai.imagengcsimage.md#imagengcsimagegcsuri) | string | The URI of the file stored in a Cloud Storage for Firebase bucket. | -| [mimeType](./ai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). | +| [gcsURI](./ai.imagengcsimage.md#imagengcsimagegcsuri) | string | (Public Preview) The URI of the file stored in a Cloud Storage for Firebase bucket. | +| [mimeType](./ai.imagengcsimage.md#imagengcsimagemimetype) | string | (Public Preview) The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). | ## ImagenGCSImage.gcsURI +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The URI of the file stored in a Cloud Storage for Firebase bucket. Signature: @@ -43,6 +49,9 @@ gcsURI: string; ## ImagenGCSImage.mimeType +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The MIME type of the image; either `"image/png"` or `"image/jpeg"`. To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md new file mode 100644 index 00000000000..3b221933034 --- /dev/null +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -0,0 +1,51 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelCreateCoreOptions interface +(EXPERIMENTAL) Configures the creation of an on-device language model session. + +Signature: + +```typescript +export interface LanguageModelCreateCoreOptions +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [expectedInputs](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionsexpectedinputs) | [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface)\[\] | | +| [temperature](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstemperature) | number | | +| [topK](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstopk) | number | | + +## LanguageModelCreateCoreOptions.expectedInputs + +Signature: + +```typescript +expectedInputs?: LanguageModelExpected[]; +``` + +## LanguageModelCreateCoreOptions.temperature + +Signature: + +```typescript +temperature?: number; +``` + +## LanguageModelCreateCoreOptions.topK + +Signature: + +```typescript +topK?: number; +``` diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md new file mode 100644 index 00000000000..5d2ec9c69ad --- /dev/null +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -0,0 +1,43 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelCreateOptions interface +(EXPERIMENTAL) Configures the creation of an on-device language model session. + +Signature: + +```typescript +export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions +``` +Extends: [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [initialPrompts](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionsinitialprompts) | [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface)\[\] | | +| [signal](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionssignal) | AbortSignal | | + +## LanguageModelCreateOptions.initialPrompts + +Signature: + +```typescript +initialPrompts?: LanguageModelMessage[]; +``` + +## LanguageModelCreateOptions.signal + +Signature: + +```typescript +signal?: AbortSignal; +``` diff --git a/docs-devsite/ai.languagemodelexpected.md b/docs-devsite/ai.languagemodelexpected.md new file mode 100644 index 00000000000..d27e718e1eb --- /dev/null +++ b/docs-devsite/ai.languagemodelexpected.md @@ -0,0 +1,42 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelExpected interface +(EXPERIMENTAL) Options for the expected inputs for an on-device language model. + +Signature: + +```typescript +export interface LanguageModelExpected +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [languages](./ai.languagemodelexpected.md#languagemodelexpectedlanguages) | string\[\] | | +| [type](./ai.languagemodelexpected.md#languagemodelexpectedtype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | + +## LanguageModelExpected.languages + +Signature: + +```typescript +languages?: string[]; +``` + +## LanguageModelExpected.type + +Signature: + +```typescript +type: LanguageModelMessageType; +``` diff --git a/docs-devsite/ai.languagemodelmessage.md b/docs-devsite/ai.languagemodelmessage.md new file mode 100644 index 00000000000..228a31c8521 --- /dev/null +++ b/docs-devsite/ai.languagemodelmessage.md @@ -0,0 +1,42 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelMessage interface +(EXPERIMENTAL) An on-device language model message. + +Signature: + +```typescript +export interface LanguageModelMessage +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [content](./ai.languagemodelmessage.md#languagemodelmessagecontent) | [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface)\[\] | | +| [role](./ai.languagemodelmessage.md#languagemodelmessagerole) | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | + +## LanguageModelMessage.content + +Signature: + +```typescript +content: LanguageModelMessageContent[]; +``` + +## LanguageModelMessage.role + +Signature: + +```typescript +role: LanguageModelMessageRole; +``` diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md new file mode 100644 index 00000000000..71d2ce9919b --- /dev/null +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -0,0 +1,42 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelMessageContent interface +(EXPERIMENTAL) An on-device language model content object. + +Signature: + +```typescript +export interface LanguageModelMessageContent +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [type](./ai.languagemodelmessagecontent.md#languagemodelmessagecontenttype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | +| [value](./ai.languagemodelmessagecontent.md#languagemodelmessagecontentvalue) | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | + +## LanguageModelMessageContent.type + +Signature: + +```typescript +type: LanguageModelMessageType; +``` + +## LanguageModelMessageContent.value + +Signature: + +```typescript +value: LanguageModelMessageContentValue; +``` diff --git a/docs-devsite/ai.languagemodelpromptoptions.md b/docs-devsite/ai.languagemodelpromptoptions.md new file mode 100644 index 00000000000..35a22c3d1a6 --- /dev/null +++ b/docs-devsite/ai.languagemodelpromptoptions.md @@ -0,0 +1,33 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelPromptOptions interface +(EXPERIMENTAL) Options for an on-device language model prompt. + +Signature: + +```typescript +export interface LanguageModelPromptOptions +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [responseConstraint](./ai.languagemodelpromptoptions.md#languagemodelpromptoptionsresponseconstraint) | object | | + +## LanguageModelPromptOptions.responseConstraint + +Signature: + +```typescript +responseConstraint?: object; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 9900b3ecccc..29b3f73f86e 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -19,7 +19,7 @@ The Firebase AI Web SDK. | function(app, ...) | | [getAI(app, options)](./ai.md#getai_a94a413) | Returns the default [AI](./ai.ai.md#ai_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. | | function(ai, ...) | -| [getGenerativeModel(ai, modelParams, requestOptions)](./ai.md#getgenerativemodel_80bd839) | Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getGenerativeModel(ai, modelParams, requestOptions)](./ai.md#getgenerativemodel_c63f46a) | Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | | [getImagenModel(ai, modelParams, requestOptions)](./ai.md#getimagenmodel_e1f6645) | (Public Preview) Returns an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | ## Classes @@ -51,6 +51,7 @@ The Firebase AI Web SDK. | [AI](./ai.ai.md#ai_interface) | An instance of the Firebase AI SDK.Do not create this instance directly. Instead, use [getAI()](./ai.md#getai_a94a413). | | [AIOptions](./ai.aioptions.md#aioptions_interface) | Options for initializing the AI service using [getAI()](./ai.md#getai_a94a413). This allows specifying which backend to use (Vertex AI Gemini API or Gemini Developer API) and configuring its specific options (like location for Vertex AI). | | [BaseParams](./ai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. | +| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.These methods should not be called directly by the user. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | @@ -81,18 +82,26 @@ The Firebase AI Web SDK. | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | -| [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | (EXPERIMENTAL) Configures hybrid inference. | +| [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | (Public Preview) An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | | [ImagenInlineImage](./ai.imageninlineimage.md#imageninlineimage_interface) | (Public Preview) An image generated by Imagen, represented as inline data. | | [ImagenModelParams](./ai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./ai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | +| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | (EXPERIMENTAL) Configures the creation of an on-device language model session. | +| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | (EXPERIMENTAL) Configures the creation of an on-device language model session. | +| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | (EXPERIMENTAL) Options for the expected inputs for an on-device language model. | +| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | (EXPERIMENTAL) An on-device language model message. | +| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | (EXPERIMENTAL) An on-device language model content object. | +| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | (EXPERIMENTAL) Options for an on-device language model prompt. | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | -| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). | -| [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of "object" when not using the Schema.object() helper. | +| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | +| [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the Schema.object() helper. | +| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | (EXPERIMENTAL) Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | -| [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). | +| [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [RetrievedContextAttribution](./ai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | | [SafetyRating](./ai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./ai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | @@ -128,6 +137,7 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | | [Modality](./ai.md#modality) | Content part modality. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -150,6 +160,10 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | +| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | +| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | | [Modality](./ai.md#modality) | Content part modality. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -211,14 +225,14 @@ const ai = getAI(app, { backend: new VertexAIBackend() }); ## function(ai, ...) -### getGenerativeModel(ai, modelParams, requestOptions) {:#getgenerativemodel_80bd839} +### getGenerativeModel(ai, modelParams, requestOptions) {:#getgenerativemodel_c63f46a} Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. Signature: ```typescript -export declare function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; ``` #### Parameters @@ -226,7 +240,7 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams, req | Parameter | Type | Description | | --- | --- | --- | | ai | [AI](./ai.ai.md#ai_interface) | | -| modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) | | +| modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) \| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | Returns: @@ -488,6 +502,20 @@ ImagenSafetyFilterLevel: { } ``` +## InferenceMode + +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +InferenceMode: { + readonly PREFER_ON_DEVICE: "prefer_on_device"; + readonly ONLY_ON_DEVICE: "only_on_device"; + readonly ONLY_IN_CLOUD: "only_in_cloud"; +} +``` + ## Modality Content part modality. @@ -590,6 +618,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason]; ## FunctionCallingMode + Signature: ```typescript @@ -693,6 +722,46 @@ Text prompts provided as inputs and images (generated or uploaded) through Image export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typeof ImagenSafetyFilterLevel]; ``` +## InferenceMode + +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; +``` + +## LanguageModelMessageContentValue + +(EXPERIMENTAL) Content formats that can be provided as on-device message content. + +Signature: + +```typescript +export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; +``` + +## LanguageModelMessageRole + +(EXPERIMENTAL) Allowable roles for on-device language model usage. + +Signature: + +```typescript +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +``` + +## LanguageModelMessageType + +(EXPERIMENTAL) Allowable types for on-device language model messages. + +Signature: + +```typescript +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; +``` + ## Modality Content part modality. diff --git a/docs-devsite/ai.modelparams.md b/docs-devsite/ai.modelparams.md index a92b2e9035d..a5722e7d69d 100644 --- a/docs-devsite/ai.modelparams.md +++ b/docs-devsite/ai.modelparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ModelParams interface -Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). +Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). Signature: diff --git a/docs-devsite/ai.objectschemarequest.md b/docs-devsite/ai.objectschemarequest.md index bde646e0ac0..267e2d43345 100644 --- a/docs-devsite/ai.objectschemarequest.md +++ b/docs-devsite/ai.objectschemarequest.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ObjectSchemaRequest interface -Interface for JSON parameters in a schema of "object" when not using the `Schema.object()` helper. +Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the `Schema.object()` helper. Signature: diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md new file mode 100644 index 00000000000..bce68ff8174 --- /dev/null +++ b/docs-devsite/ai.ondeviceparams.md @@ -0,0 +1,42 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# OnDeviceParams interface +(EXPERIMENTAL) Encapsulates configuration for on-device inference. + +Signature: + +```typescript +export interface OnDeviceParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [createOptions](./ai.ondeviceparams.md#ondeviceparamscreateoptions) | [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | +| [promptOptions](./ai.ondeviceparams.md#ondeviceparamspromptoptions) | [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | | + +## OnDeviceParams.createOptions + +Signature: + +```typescript +createOptions?: LanguageModelCreateOptions; +``` + +## OnDeviceParams.promptOptions + +Signature: + +```typescript +promptOptions?: LanguageModelPromptOptions; +``` diff --git a/docs-devsite/ai.requestoptions.md b/docs-devsite/ai.requestoptions.md index 73aa03c1d25..8178ef5b696 100644 --- a/docs-devsite/ai.requestoptions.md +++ b/docs-devsite/ai.requestoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # RequestOptions interface -Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). +Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). Signature: diff --git a/packages/ai/src/api.test.ts b/packages/ai/src/api.test.ts index 27237b4edd3..76a9b4523c2 100644 --- a/packages/ai/src/api.test.ts +++ b/packages/ai/src/api.test.ts @@ -21,7 +21,7 @@ import { expect } from 'chai'; import { AI } from './public-types'; import { GenerativeModel } from './models/generative-model'; import { VertexAIBackend } from './backend'; -import { AI_TYPE } from './constants'; +import { AI_TYPE, DEFAULT_HYBRID_IN_CLOUD_MODEL } from './constants'; const fakeAI: AI = { app: { @@ -102,6 +102,21 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getGenerativeModel with HybridParams sets a default model', () => { + const genModel = getGenerativeModel(fakeAI, { + mode: 'only_on_device' + }); + expect(genModel.model).to.equal( + `publishers/google/models/${DEFAULT_HYBRID_IN_CLOUD_MODEL}` + ); + }); + it('getGenerativeModel with HybridParams honors a model override', () => { + const genModel = getGenerativeModel(fakeAI, { + mode: 'prefer_on_device', + inCloudParams: { model: 'my-model' } + }); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); it('getImagenModel throws if no model is provided', () => { try { getImagenModel(fakeAI, {} as ImagenModelParams); diff --git a/packages/ai/src/api.ts b/packages/ai/src/api.ts index e9264262145..62c7c27f07a 100644 --- a/packages/ai/src/api.ts +++ b/packages/ai/src/api.ts @@ -18,11 +18,12 @@ import { FirebaseApp, getApp, _getProvider } from '@firebase/app'; import { Provider } from '@firebase/component'; import { getModularInstance } from '@firebase/util'; -import { AI_TYPE } from './constants'; +import { AI_TYPE, DEFAULT_HYBRID_IN_CLOUD_MODEL } from './constants'; import { AIService } from './service'; import { AI, AIOptions } from './public-types'; import { ImagenModelParams, + HybridParams, ModelParams, RequestOptions, AIErrorCode @@ -31,6 +32,8 @@ import { AIError } from './errors'; import { AIModel, GenerativeModel, ImagenModel } from './models'; import { encodeInstanceIdentifier } from './helpers'; import { GoogleAIBackend } from './backend'; +import { ChromeAdapterImpl } from './methods/chrome-adapter'; +import { LanguageModel } from './types/language-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; @@ -94,16 +97,36 @@ export function getAI( */ export function getGenerativeModel( ai: AI, - modelParams: ModelParams, + modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions ): GenerativeModel { - if (!modelParams.model) { + // Uses the existence of HybridParams.mode to clarify the type of the modelParams input. + const hybridParams = modelParams as HybridParams; + let inCloudParams: ModelParams; + if (hybridParams.mode) { + inCloudParams = hybridParams.inCloudParams || { + model: DEFAULT_HYBRID_IN_CLOUD_MODEL + }; + } else { + inCloudParams = modelParams as ModelParams; + } + + if (!inCloudParams.model) { throw new AIError( AIErrorCode.NO_MODEL, `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })` ); } - return new GenerativeModel(ai, modelParams, requestOptions); + let chromeAdapter: ChromeAdapterImpl | undefined; + // Do not initialize a ChromeAdapter if we are not in hybrid mode. + if (typeof window !== 'undefined' && hybridParams.mode) { + chromeAdapter = new ChromeAdapterImpl( + window.LanguageModel as LanguageModel, + hybridParams.mode, + hybridParams.onDeviceParams + ); + } + return new GenerativeModel(ai, inCloudParams, requestOptions, chromeAdapter); } /** diff --git a/packages/ai/src/constants.ts b/packages/ai/src/constants.ts index cb54567735a..b6bd8e220ad 100644 --- a/packages/ai/src/constants.ts +++ b/packages/ai/src/constants.ts @@ -30,3 +30,8 @@ export const PACKAGE_VERSION = version; export const LANGUAGE_TAG = 'gl-js'; export const DEFAULT_FETCH_TIMEOUT_MS = 180 * 1000; + +/** + * Defines the name of the default in-cloud model to use for hybrid inference. + */ +export const DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite'; diff --git a/packages/ai/src/methods/chat-session.test.ts b/packages/ai/src/methods/chat-session.test.ts index 0564aa84ed6..f523672f5e2 100644 --- a/packages/ai/src/methods/chat-session.test.ts +++ b/packages/ai/src/methods/chat-session.test.ts @@ -20,10 +20,11 @@ import { match, restore, stub, useFakeTimers } from 'sinon'; import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; import * as generateContentMethods from './generate-content'; -import { GenerateContentStreamResult } from '../types'; +import { GenerateContentStreamResult, InferenceMode } from '../types'; import { ChatSession } from './chat-session'; import { ApiSettings } from '../types/internal'; import { VertexAIBackend } from '../backend'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -36,6 +37,12 @@ const fakeApiSettings: ApiSettings = { backend: new VertexAIBackend() }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('ChatSession', () => { afterEach(() => { restore(); @@ -46,7 +53,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContent' ).rejects('generateContent failed'); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + fakeChromeAdapter + ); await expect(chatSession.sendMessage('hello')).to.be.rejected; expect(generateContentStub).to.be.calledWith( fakeApiSettings, @@ -63,7 +74,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContentStream' ).rejects('generateContentStream failed'); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + fakeChromeAdapter + ); await expect(chatSession.sendMessageStream('hello')).to.be.rejected; expect(generateContentStreamStub).to.be.calledWith( fakeApiSettings, @@ -82,7 +97,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContentStream' ).resolves({} as unknown as GenerateContentStreamResult); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + fakeChromeAdapter + ); await chatSession.sendMessageStream('hello'); expect(generateContentStreamStub).to.be.calledWith( fakeApiSettings, diff --git a/packages/ai/src/methods/chat-session.ts b/packages/ai/src/methods/chat-session.ts index 60794001e37..dac16430b7a 100644 --- a/packages/ai/src/methods/chat-session.ts +++ b/packages/ai/src/methods/chat-session.ts @@ -30,6 +30,7 @@ import { validateChatHistory } from './chat-session-helpers'; import { generateContent, generateContentStream } from './generate-content'; import { ApiSettings } from '../types/internal'; import { logger } from '../logger'; +import { ChromeAdapter } from '../types/chrome-adapter'; /** * Do not log a message for this error. @@ -50,6 +51,7 @@ export class ChatSession { constructor( apiSettings: ApiSettings, public model: string, + private chromeAdapter?: ChromeAdapter, public params?: StartChatParams, public requestOptions?: RequestOptions ) { @@ -95,6 +97,7 @@ export class ChatSession { this._apiSettings, this.model, generateContentRequest, + this.chromeAdapter, this.requestOptions ) ) @@ -146,6 +149,7 @@ export class ChatSession { this._apiSettings, this.model, generateContentRequest, + this.chromeAdapter, this.requestOptions ); diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts new file mode 100644 index 00000000000..83610f3dcd6 --- /dev/null +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -0,0 +1,776 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AIError } from '../errors'; +import { expect, use } from 'chai'; +import sinonChai from 'sinon-chai'; +import chaiAsPromised from 'chai-as-promised'; +import { ChromeAdapterImpl } from './chrome-adapter'; +import { + Availability, + LanguageModel, + LanguageModelCreateOptions, + LanguageModelMessage +} from '../types/language-model'; +import { match, stub } from 'sinon'; +import { GenerateContentRequest, AIErrorCode, InferenceMode } from '../types'; +import { Schema } from '../api'; + +use(sinonChai); +use(chaiAsPromised); + +/** + * Converts the ReadableStream from response.body to an array of strings. + */ +async function toStringArray( + stream: ReadableStream +): Promise { + const decoder = new TextDecoder(); + const actual = []; + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + actual.push(decoder.decode(value)); + } + return actual; +} + +describe('ChromeAdapter', () => { + describe('constructor', () => { + it('sets image as expected input type by default', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.AVAILABLE) + } as LanguageModel; + const availabilityStub = stub( + languageModelProvider, + 'availability' + ).resolves(Availability.AVAILABLE); + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'hi' }] + } + ] + }); + expect(availabilityStub).to.have.been.calledWith({ + expectedInputs: [{ type: 'image' }] + }); + }); + it('honors explicitly set expected inputs', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.AVAILABLE) + } as LanguageModel; + const availabilityStub = stub( + languageModelProvider, + 'availability' + ).resolves(Availability.AVAILABLE); + const createOptions = { + // Explicitly sets expected inputs. + expectedInputs: [{ type: 'text' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { + createOptions + } + ); + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'hi' }] + } + ] + }); + expect(availabilityStub).to.have.been.calledWith(createOptions); + }); + }); + describe('isAvailable', () => { + it('returns false if mode is only cloud', async () => { + const adapter = new ChromeAdapterImpl( + {} as LanguageModel, + InferenceMode.ONLY_IN_CLOUD + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns true if mode is only on device and is available', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.AVAILABLE + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.true; + }); + it('throws if mode is only on device and is unavailable', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.UNAVAILABLE + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + await expect( + adapter.isAvailable({ + contents: [] + }) + ).to.be.rejected; + }); + it('returns true after waiting for download if mode is only on device', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.DOWNLOADING, + create: ({}: LanguageModelCreateOptions) => + Promise.resolve({} as LanguageModel) + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.true; + }); + it('returns false if LanguageModel API is undefined', async () => { + const adapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request contents empty', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.AVAILABLE + } as LanguageModel, + InferenceMode.PREFER_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request content has "function" role', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.AVAILABLE + } as LanguageModel, + InferenceMode.PREFER_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'function', + parts: [] + } + ] + }) + ).to.be.false; + }); + it('returns true if request has image with supported mime type', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.AVAILABLE + } as LanguageModel, + InferenceMode.PREFER_ON_DEVICE + ); + for (const mimeType of ChromeAdapterImpl.SUPPORTED_MIME_TYPES) { + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [ + { + inlineData: { + mimeType, + data: '' + } + } + ] + } + ] + }) + ).to.be.true; + } + }); + it('returns true if model is readily available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.AVAILABLE) + } as LanguageModel; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [ + { text: 'describe this image' }, + { inlineData: { mimeType: 'image/jpeg', data: 'asd' } } + ] + } + ] + }) + ).to.be.true; + }); + it('returns false and triggers download when model is available after download', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.DOWNLOADABLE), + create: () => Promise.resolve({}) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + {} as LanguageModel + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { createOptions } + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + expect(createStub).to.have.been.calledOnceWith(createOptions); + }); + it('avoids redundant downloads', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.DOWNLOADABLE), + create: () => Promise.resolve({}) + } as LanguageModel; + const downloadPromise = new Promise(() => { + /* never resolves */ + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledOnce; + }); + it('clears state when download completes', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.DOWNLOADABLE), + create: () => Promise.resolve({}) + } as LanguageModel; + let resolveDownload; + const downloadPromise = new Promise(resolveCallback => { + resolveDownload = resolveCallback; + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + resolveDownload!(); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledTwice; + }); + it('returns false when model is never available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.UNAVAILABLE), + create: () => Promise.resolve({}) + } as LanguageModel; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + }); + }); + describe('generateContent', () => { + it('throws if Chrome API is undefined', async () => { + const adapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.ONLY_ON_DEVICE + ); + await expect( + adapter.generateContent({ + contents: [] + }) + ) + .to.eventually.be.rejectedWith( + AIError, + 'Chrome AI requested for unsupported browser version.' + ) + .and.have.property('code', AIErrorCode.UNSUPPORTED); + }); + it('generates content', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const promptOutput = 'hi'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const createOptions = { + systemPrompt: 'be yourself', + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { createOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + const response = await adapter.generateContent(request); + // Asserts initialization params are proxied. + expect(createStub).to.have.been.calledOnceWith(createOptions); + // Asserts Vertex input type is mapped to Chrome type. + expect(promptStub).to.have.been.calledOnceWith([ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + // Asserts expected output. + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: promptOutput }] + } + } + ] + }); + }); + it('generates content using image type input', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const promptOutput = 'hi'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const createOptions = { + systemPrompt: 'be yourself', + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { createOptions } + ); + const request = { + contents: [ + { + role: 'user', + parts: [ + { text: 'anything' }, + { + inlineData: { + data: sampleBase64EncodedImage, + mimeType: 'image/jpeg' + } + } + ] + } + ] + } as GenerateContentRequest; + const response = await adapter.generateContent(request); + // Asserts initialization params are proxied. + expect(createStub).to.have.been.calledOnceWith(createOptions); + // Asserts Vertex input type is mapped to Chrome type. + expect(promptStub).to.have.been.calledOnceWith([ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + }, + { + type: 'image', + value: match.instanceOf(ImageBitmap) + } + ] + } + ]); + // Asserts expected output. + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: promptOutput }] + } + } + ] + }); + }); + it('honors prompt options', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') + } as LanguageModel; + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const promptOutput = '{}'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const promptOptions = { + responseConstraint: Schema.object({ + properties: {} + }) + }; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { promptOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + await adapter.generateContent(request); + expect(promptStub).to.have.been.calledOnceWith( + [ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ], + promptOptions + ); + }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('unused') + } as LanguageModel; + const promptStub = stub(languageModel, 'prompt').resolves('unused'); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContent(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + }); + }); + describe('countTokens', () => { + it('counts tokens is not yet available', async () => { + const inputText = 'first'; + // setting up stubs + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + measureInputUsage: _i => Promise.resolve(123) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + + const countTokenRequest = { + contents: [{ role: 'user', parts: [{ text: inputText }] }] + } as GenerateContentRequest; + + try { + await adapter.countTokens(countTokenRequest); + } catch (e) { + // the call to countToken should be rejected with Error + expect((e as AIError).code).to.equal(AIErrorCode.REQUEST_ERROR); + expect((e as AIError).message).includes('not yet available'); + } + + // Asserts that no language model was initialized + expect(createStub).not.called; + }); + }); + describe('generateContentStream', () => { + it('generates content stream', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + promptStreaming: _i => new ReadableStream() + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const part = 'hi'; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream({ + start(controller) { + controller.enqueue([part]); + controller.close(); + } + }) + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { createOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + const response = await adapter.generateContentStream(request); + expect(createStub).to.have.been.calledOnceWith(createOptions); + expect(promptStub).to.have.been.calledOnceWith([ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + const actual = await toStringArray(response.body!); + expect(actual).to.deep.equal([ + `data: {"candidates":[{"content":{"role":"model","parts":[{"text":["${part}"]}]}}]}\n\n` + ]); + }); + it('generates content stream with image input', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + promptStreaming: _i => new ReadableStream() + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const part = 'hi'; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream({ + start(controller) { + controller.enqueue([part]); + controller.close(); + } + }) + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { createOptions } + ); + const request = { + contents: [ + { + role: 'user', + parts: [ + { text: 'anything' }, + { + inlineData: { + data: sampleBase64EncodedImage, + mimeType: 'image/jpeg' + } + } + ] + } + ] + } as GenerateContentRequest; + const response = await adapter.generateContentStream(request); + expect(createStub).to.have.been.calledOnceWith(createOptions); + expect(promptStub).to.have.been.calledOnceWith([ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + }, + { + type: 'image', + value: match.instanceOf(ImageBitmap) + } + ] + } + ]); + const actual = await toStringArray(response.body!); + expect(actual).to.deep.equal([ + `data: {"candidates":[{"content":{"role":"model","parts":[{"text":["${part}"]}]}}]}\n\n` + ]); + }); + it('honors prompt options', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + promptStreaming: p => new ReadableStream() + } as LanguageModel; + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream() + ); + const promptOptions = { + responseConstraint: Schema.object({ + properties: {} + }) + }; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE, + { promptOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + await adapter.generateContentStream(request); + expect(promptStub).to.have.been.calledOnceWith( + [ + { + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ], + promptOptions + ); + }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + promptStreaming: p => new ReadableStream() + } as LanguageModel; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream() + ); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapterImpl( + languageModelProvider, + InferenceMode.PREFER_ON_DEVICE + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContentStream(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + }); + }); +}); + +// TODO: Move to using image from test-utils. +const sampleBase64EncodedImage = + '/9j/4QDeRXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwABAAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAABIAAAAAQAAAEgAAAABAAAABwAAkAcABAAAADAyMTABkQcABAAAAAECAwCGkgcAFgAAAMAAAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAAMgAAAADoAQAAQAAACwBAAAAAAAAQVNDSUkAAABQaWNzdW0gSUQ6IDM5MP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/CABEIASwAyAMBIgACEQEDEQH/xAAbAAABBQEBAAAAAAAAAAAAAAAAAQIDBAUGB//EABgBAQEBAQEAAAAAAAAAAAAAAAABAgME/9oADAMBAAIQAxAAAAHfA7ZFFgBQAAUUBQFBFABSUBQBQBZQUiqC7wAoigooQKACgCigKIoAosIKSigABWBdZAUAUAUQUUUAFIBQAWAFAUVFABSKoLqAKAKAKJVt4BvrFLAqKooArHgoQAoKiqDyKKoaiqhSqhCqgLFKHKdBiZmbodX5n2MbWHkdZS2kWhUBQIVUBwgUucv8Oad7nUzey3vPO5q4UrlOEWjzT0vhssDpea9Gy03BsqooKhCgCgCgHIcd0fN5DnuWHseY0Ureh+ZelLIqFq+f+gQJ5f6V5r6pE4i2ioDhCFVAVWrCiBxvJdlzFzVc56GjFoy4/a8d2q2TmpN3V1OF2MWp1/NrL0hzinRnO5Sdwc+L0Jz5HQLzyy9AYQYmDrZfXkyxVs5m4yVt3F0/M7l1YotpQnScdumqsFSb0yElm4zf5hjvV56bOtteViXq3ecRMbJgG+L4tzGqNyTDJNqMx5rfSHGRdpAcidPqLyFbuBeWrdmyONg7TJTBTrqZg3b6GGzbSzILYW8uSuF2hPG9l6uFdbPQRxzU8M2Lc62fpUJZNGC5TXAseNuVc2abO0pSKUsjdI+OdNoTzYc3fIANzF1LVTalK9KU72e1coa1TOqe3naA8inKGZ0QV5ZGzSywKWVrSAUROTjuno8lSLQbFq5kNrXsYAvQu5xmW9y18l0tjmrFu8ZM66C0nLabEsPGrT3xOlnIyXjkzC8tSxh2zRbWlsVNZtY6a9SKq1ZCd0rLHS17SPlgUtvpvatrVetlYJJZRpNcOOfmRaEN+s3Vctl0qCWs+PLljs19iWw+RdZEcU1VBFVUR6Kr5a6rplEzvnH5krF9Y33LnNFkqWIynAqZ3Zno3U03xO1mVY1HrGDxgOREpURkjiMXDUXOlsVpjRIJ0RXhix3KbUuzn6DLla6nK1RwFAKKK+GNsuigXReXW6mpRS2yWu6Zgr64Rq90abqclllYVJiJxIrAkI1JXRvJZoJJqUcY1yzmrvLnMLJX1QngWQrF9hTW01IZmwlt1F5bWtMTPruLc+fYltSVo83SKpnX/8QALRAAAQQCAQMDBAIBBQAAAAAAAQACAwQREgUQExQgITAVIjEyI0AkJTM0QXD/2gAIAQEAAQUC/wDH5Z2wu/scrHmBjg+P0hzXf0pGCSPjpnwT2bDa0LOWe6dEgCW06yYIWwRf0uVrbNdf79Grg2ZeUrxkMsco+CFleP4uRuyQvPITOjdyLzS4yy+Znqts7dtcbSZOgAB8V6Yw1nlziCE39obclR8EzZ4YrUM7vRy2PLVBpbT+Plv+Nn0RPZU42jJpc9HIwOhtqk8yU/j5dxMq+1YbrVaH2eUd/lsDpJG516zRMnjLSHRt0i+PlYss613Fli5OLBhOkwv1ShNG4PlDIqdzyunjd/l/k5NwFWu0dw/gMLlXhfFyHLD+SpGZbTq8GIR3Y7NCGKvRrd9fT5F4VgLxboXZ5ALXkgs8mFZt3I5vIvLzLYXnzL6lhfVYwvq9dfVqy5IEpzTG93618me0P9S5T96GPNQDWm+f8HifZuVlZWVlZXJnPILKysoytXsuUe0y27LHxzS92Y/ca72xzmWOW1cMcklSSKIMkbIzzYNrs8b6dO1HXYLsBaHAqS0yOTKyvLb37crZOQm5Bkcw5GFykuyqZ81iJ0mru9JgJ8bmHoGly1ds+KSNMikkXZsAduVo+5HKBwmW5mFzy5z70r43WJXEyuKz9ywjs8wzSQPdkuwUAcch/u9InavA0s2maqnMYpC1rmtjAV1zvHpVi1hiiQghz4cC8SsnUqxX0+svDrix9KgzLxeHHiiG/SX4+lyI8ZMFLVmgFz9nY2UELioNnqSRz5KEa/6AUpe0Miyrf8Dadnug6uQwOjgSyKye+WyIbAEgLuRoSxORwVLU2tTyOfJj2QlkY3ua8dGN0MhO2LmkK3bkgn7Ykjk4+KQ14BXj67YNkydqtE/VahagLVqwFo3f0PHlwe4NOSWRrh7agqxUEyZmGF9+IKG/G53Q7YPfaou9amEzV+wAI9BkY0k5PWtHOwy1d3V4zC38oKaq6WQfiw+FrIIqxXutiPRlfatWLVi0YvZTU4bDnVV4zkKpRrvUbS1F3tG4hbhbhbhS2WxtmmM0nHt0gysrZZWfR7rPXKysrZbFblblbruFZ990Nc7BCYpsxXdXcWy2WyysrPXuxrvMK7sa1ytF212120RqMZGFhY6BAoFArZZWVlZWfTC1zi+0c15y9+q1WgT4F33KOUl+0a7jMtfl2PTn4K+S0xPDoIe2srKyrE2vSGPuP7LF22/EEFq5dtybDlMAYMrZbLdOsgJ7t3KJj4xn4crK2QkKDgfTnpMThmNU1jXMbNogc/DlZWVno1+FsAvz6H5x0/KhZ7/GR0wgPd7tjD1x0f8Auoxs/wCHCwtemOuUx4ag8FZHV8bcqu33+LKysArt5WpWq1WOmShIQnSZBTBs4eyz1z8AKygvZaharC1RYsdQcESLcL8rJWVn0Z6gdG9MrKys9CAUWLtuWvUEhCRbDp7rZbLKCCygvx6s9AUCisBYRCPTKyUPQ0ooOKBK/8QAIhEAAwACAgIBBQAAAAAAAAAAAAEREBIgIQIwURMiMUBQ/9oACAEDAQE/Af5k9E9yWITC9S7RCCIQhCEGuyEcPFMTYrCYsxTrDYmVQTKhPouPJ9GyNj6iG7mEIRkZGPxZGR8aTofiRkZGM6OjY/OahNFp38lZWX5NkXxPtxuzZlNjZm5ubmxc01RqakIak4XhSl9NJxf6cJxvNCxCelMp/8QAIhEAAwACAgIBBQAAAAAAAAAAAAERECASMAIhIjFAQVBx/9oACAECAQE/Af1d6LumXZs5MTLhn51pR5WlKUulz5JLFLrR/XH8ITEIQhCCHld3IbRUesez2Px0jI8PERxIz5HyPZxRxWkIQmvI5FLil6Z137C9NJ2XFL0MhD//xAA2EAABAwEFBQcDBAEFAAAAAAABAAIRIQMQEjFBEyAiMlEEMDNSYXGRQIGhIzRCklAUQ1Nwcv/aAAgBAQAGPwL/AKfYHfyMfUttf+M1TXNyIpvHCQY+icw5OEI9ktdKBbR3sAmjZDZkxnW6TQI2HZK+a00CDG/Ri3Zm3mjonWNtGMZOTJgCdTCIaS8+ixOOCyCDLMU7sWVnQxJKaHEyMy2kqWyLSYxJwtHS5u/atiOK5z7USGmIQAHdktMONAsTnEn1WQKnojgjCdE21FAUW2b5I3aHStzZ1r3jP/d5uDbV1XyWgKzrAy3Xn+L+IXWTj5e8s2aRN2SOhVm1woXLDo1oQazmOSGLOK7hY9shYdckxvQDvGWvQxuMeBiIOSbNjs36kpjvKZXihSHhOfnhE0TuDDHrdaECGMdLu9w6khYncrBiKlBozJhWTHiHAqyd6Qms+VJsmfCwhh9k97C8EDqn/quZHlVO2Wi4e2OVO2KnamrxbIr/AGimi0OA9GL9qFXsZVeyPVezWirY2qq20H2Wbv6qy+E5hzFEFZgecKwI1Vh91bOGmV1B6K1Vr9t9vsN3mCqAm7N7SOjdE0NqQZTrTrc1ztCrJ4PC3VWDcQnF+FbvLhzfhYmmicMfKuF04skQ+eI6LFtBms0xhNXH4v2MVWIHhELCDiGvoqHWE6rWwadUHTJb5dQuE16ojaEjOt0OEX0ErDBk6IF7YnqjgYTGcLw3wpwOj2WqqFTNE4qnOViJWCaR0VXnKKKr/wAKTfJMlTEjVsolZXNoAIzRuBmEHWwaGnJzRRbTZ8PnCLZaGn0WS5KrCLM1WK0xD0OS8Jhn0RH+nZ/VeC1eC1eEFyflYHWsTkAuZ/yoZaf2Xij7hTtW/YLnb+Vzs+VLsvRybaEV6SjhENu2kNwN8yfbFoMcrf4p1o9pwikTQIl1nXQkXVXCGhYiYJ8rl+4tGTlAR5nR/IthQVS4j4WztHEnQlgVLX5YtFUwvFHyqWjflcy2r3WZZ5SjifiAyXpdha8hvRCGzwprA0kzWEABT3XCQPcKpCwsIy6IY/xRTjeD7ysAM+u5ov07LaHoVithx9JyvoB8LIfCyU7Ie+60sPG3MXHEeEZIVr7qoaUDQP6obR0x0CptPhBhDhN9Ci9xDoya0IutHusmt/iFBIXDakey8QlZ31c0fdTuY2wAeqxC0OI5yoxk+l+MWpb6XfrAV0WOyAprcOAn23ch8LLcxPxfK4XfKzCqVkhxqhquMrNZrNTzegWM0U6uP00rJThF2ar3WfdSPo5mAFDcuqwu3JYYN3EQAuZRKw4e+e3QhYYWI825hGt0aLJZd5kslxKBu5IuN2hnvc+4gIzdzQVhNfX6CqpuZX0VR39d83D6ckG7F/kafT0/xf8A/8QAKhABAAIBAwMDBAIDAQAAAAAAAQARITFBURBhcSCBkTChscHR8EBQ4fH/2gAIAQEAAT8h/wAiv8iof60/24fSvm0naH+R2aUdppQR8PVerRTWafXUA+lrvlRRsJt2f+xcK5o6rMHN0LZb9Fagaq0EyEPYezzAGwavL67l+jb1sex1ucH2lNKQvo1+4DXUq1qO8JQuOPmZPNWNPbllNUa93l+m+Nx3niXqZkfLEtIvwwS75Bt1qXL9H43mjIKjs5hxLIxhtWEwAKAMH07uBuNpYwtVXCGs7xLQcmZjdZmpBJoLnaFJ1hXpOcFSE2YaxxFP5/qcz+iXToFmTpK7yt+RC1GWVyrPaHXZjILVX8kNe0A+l+w+psg/PfTViLG0CD8QCO8wRgYDiC7aYcs8evd6Brtt3jBCFweZUJVb7fUI7W74YEcS8LFVhJzjk4dy8SodQh3BdmyEXRzd7TFspRGYByYeUzF14jPPEuXLly5cuX1voJWze2sQ9Q9zg+amaprCQ2IEoCSuY63Ir4MUahd+BmIVIZuUJECnsXWXLxBDX26+XmU6Xz/7B6iXK05n8hGGqPmbfyP/ACbwnQ2SxsPmU6p4Z+gVlGn8XL6L7f8AJtJ7Q/KUi17sMo5YxypaCW4JWPpGGnmOw2v8iFmYsfKLYjkdZeDFDDg0nxh+YLPL+3rAovb+8vPUvzA65saxNfuiJo4RLXF13F2lmFXuvaKkPabIc4ZYEFrumMtNnH9E5U7Xd/MEFXvNB7FuMe0c02mB3mVhstCBhU0/pNAtCaNTXRMJW6svWpfUs6vbSB84N+NZSDuiCsttdle72mPNFBy4gHLLvAbbzAzStbf3M1+rqfeaZZioic9GqZcBKxw6mYehtWyxgJ6A0l8UrYI2w+TpmbVfCc8e01A7G4Am8NmW9XzxHqqqOF68w02AWwwaR0UXXYymRduZhOHzFc3L8ydyHa660DiXiJbc7qbQ68TJeQN5lUp3IxjxlldJXAGhvzGQDjQla/mO1nlbX8SpaWtplxI3wfuMXhYM1gea6UwzwhqIoFb6IX3dfboerh4s/c7Ku7jYbcZBKfAP4hEIvg/xCqWcYJrnusF0L2ilrPtY/UeCdwsCgzQq1kzPaNZXE8vB0QuFCtP2R/SzWKmP5lZq66aINj8zdH3JY2L3b/EUWNVZT7SgKpYEv6iCaNkipsd5QBFfMK7/ADLhKuriEWio7PmWrwcAzdF4xALHlbKs4Z1wsK+kLuRnGtlWvBMmobbEsBvLa4Ra2bGWPmIdgfeWyhbQxMealG6ViFVJbmACj/e8MOBdG1M5KoWzlPfQP2TdqXYgVMbhBCOIfJjqCjWwEDunsDxEaxiLGc+YGofiC6/tph0fEbq08FzOOphG5asjVVFSkYRPapngwWxcu0vBdTFabfWF2AxjqRcMdpCHIuhjHRaq1shjR+YLyRaBfeDFw3B95hI3XGcc98n5iGQXeCM9ykB5sGtyXMwjvSacC9j0UgA0epLcxoY1vwIuGsVEyJgECgfuUxBo3SqX0bqmOle5Fwz9XSSp7y5TclPW+DjyysaQ2D7yoIZQUVASNWtGaMDyJZG1bMueKBkF4emONKdQe8fmlpZKmGwDaCjdRVzyl+r5RZctlwODPeW5l5eWnej0a07kyste7Cuz4iOp+IbRXiF0fvmcLfaBgGB59RCuYRi1grWpmq3zACxuMsW4ipmHSFCF5eEAxPoFO6HfPOX6g+h0Hr241UgcciUSu9EJR2iYsUkpMCjTWLHiCiA7Cd0TDl5ljaUzMJfQMGEBfQvMZ3mqnuQnZf4ej09wdMswMrA4BbDfiY6VK6VAgQ6e2d5Ei4qWqn5s+itCbuWLqhlWkq2LKEXLOty5cvqlICFMPQZcHouVl00QXXQwuRGdtTZDAmnruX12bcwwxnnJGlohhFSuj0Ybtvo6KU/mKNxw06XL6X6UuLMxjxEbIUS+eOldNT7zpWodT1r8S0So9Fsy1mBrWLawbfpjeawPRVbNOteu6hB2RJpKbpkjKiWOgWj0pKSXuUpKCg6bJfRcuX1GX0CxLzOdyKnhMtou0sa9L5JmoXcg2sE0PQOcoy+lstCp7dIO81QWXhJAJh0Zhme2lG0EaxxLeickGmHRljeW3gYGMiJWUqDT0rLS24nU3GkrAgLhBQ5orOopHhhHWKMs/9oADAMBAAIAAwAAABASIMVBgAVIggAJsGy6fNBiyj4Y5ptsnyTbFtvCz9pNNPGuqMCNo42YQIEExL6CRYMEGT8YCBzUGdVEHKQHraFgCRaW/wDNpnycuGNdceiyLtY4mcgOiOu29EEGuHlAnRrvBwEb0uqOJE43dRwqzkz2egbGwwUOslkwzPIcsSwSNhRUkWEw1v62L+JMcNPr2AmjywACL2YgqfCuq0/Cz+/jqnaGEcefx1OE4WV4cia8oyMQ8U8lMsIgsWO//8QAHREAAwACAwEBAAAAAAAAAAAAAAERECEgMVFBMP/aAAgBAwEBPxBc1+a/BIhCcITMI8QhCYQhCEJkvMQmYQhMwSNeZGhNUhCEIQb2JLs6VO48HoK5+AEVawVlRxOosomXwd8GnZFXhBRoo6jcWhEUOTSFpEsbUKcC6hquh+Q9qiTHo2Gy+i7hlYQVKEyMkG6xMadEsQVNWsKSdaxKa3svsSIaTUmSLsaJEyxoR7dxN2w294KG1dcCJhIQvQkXwVG3IpKLNtFFEf038E3ME6JsbQ4LKEhtzEIQgmkJBlpkEt46D4xkZcREF0PMJiix8T5k1yH+A//EAB4RAAMBAQADAQEBAAAAAAAAAAABERAhIDFBMFFh/9oACAECAQE/EPwf5PaPLlKXwo8u0pSlHxtGUpcdGmMo/RWlC6rOhZS5zhwLrp0UmC+CpFGXTp0aFzo0Khvgvd8QpR+8Uo8UY3hhO7WUKvQfs9qhB/Q1cMLofRRZwoyLzYIjmNwtyoqx5BNoX9YkbbejnwfUEgxiqXWPwCf4cfBQoKFzOCBKesbMOHCLwvBFnCFFE4bIRBUylKUqIyEEGxKimUpcjwmijeLKUuVFHlekUospdpk/Fii0nkmn/8QAJhABAAICAgICAgIDAQAAAAAAAQARITFBURBhcYGRobHBINHw4f/aAAgBAQABPxDweDX+J4P8jfk14NeVQJUNf4G/J4NeKleKh4JQyvDDwHipXivFQJUJUrxUrxUDuVK8ceArxUJUqVA8HioeK8VAzKglSoVUqVDLKhiV4rzUCoFwxKlSpXgPBAuVK8VKrwF+K8VApm5UCV4rxmVCVA81KlngPAY8V4qV1L8DfCB7N8RCCVTnDfgMeK8G5UJXgPJhh5NeefBszFrbCQytzUeUao/D74+vBr/AgAyf4TDfk8BC0HvMPJrzz5Du/sDX4afqAmGh09Z6tZ8y6HhnL0DxVZuAzNHW4FtX6iIo7J/LlggsaQei6lY9npH/AFNo2ptfvweTUuoeUhnWfias6ur9zmvJvwbOtJ6ixUpjK35UfuXT0sbc6a5cGnnUL5mcCXrzLchY3eC3HuH3Uh0/D9mofTOTtN9iw35PBr/Ac8U7vqA+qD5uBejEvV1kHSBKE5R22G1rFxXpUFJYPmYeA58heEtci8c45jURYWjAr6YsPtTBr6p1QtXvZiUhnAA9EqG/BL8GvF+HPAhZtt/Ep6IEFjWWXZEyZxhjcAsIVY6kJuM7G4jJYFaxpL6xBJXdgs7L3DZCXPuskrndJk1KfdVNat1CRLa/LF/QQxLhuX4PA/4VRxeHLBSZcWf99S27qvcugnIGo2dXu2sS82b2g/GU/MunLN0XKR9RXnZipcJeTeMnCR4FO+1/In8VEYLeinvEoIwVXoGXnxcJcGpfi/Fy21LB7I/QfuXRjHXqK8gK5zKKcge5qpOkLtH81MXGMwG1V9/qBRMNPJuMY1SJ6Zg5lwzDEepTJTCOyvUSXhBnJM/khigpQ1Qv9+L8DDEuGZcuXLmJy595j8JEMc8nuC1NlOYZQwYgoYo0vrHxDJYqMeAChgzKA1gouBzr1iKCjyip+TcPydMB03LYrV5B7uOogpwsP/EaDsTkPzzK6RwxgYYzbLC2ZleUPuA7/crA3mse/AtMIMvwuKgIR/JSndEl3GvmUJdIWrx7blVdY7bq36i1x4YU2iJHJpkW20V/ZNdWx0Fv1REywUgayt8QlCxGmUPVal73duXYUnWY+VQ5Vkvp1Ag0hWzxDsCsXKtreYa0/wDbifph/wDkpH0qKek5slT+CIaofwlXT1a/9MP+GH5h/wB0PqaXb0oftGVjP1D/ALmeGP0e9zIIYbq2kjuNCnKUn9MAvw3aQZgIXxSv8XKN2Iv0f+yWSW7IOyCu8DX+CATBIHSMWMyI3ofUAs5L8mJc6D+IMN6h7ePz/cKYvEpSSoVxhPc7rmPMHW38zcW1eWqOWAiW1MVH4jixHSNPq63CEMEwbVAtddYleJbjRl+6qUt1UOMD8x6hdbNH3OdTEKNn3uYnWIotw22VL6i1l282Y3BCipGSWhRzahznsOD76iAbC4lVV25rqG3MRWFkeviCur66Mct/MICcbEf7V7ghVYEpzTpqFMewB7H7lg2lxHBUByqDApdpbLOHlsg7m7CgEPbvqc3VboZs7UcmYEolD8gcGV/UE4ubQVrDspUiXl23DrBwRa6lX2IrB2HTqLvOkKi3pemJetOKgvvC7GOIgruagHj22wp4akoviWsDVT8BmYYyWD9LnBBXAfoYpCBtFdrgibPAo/mGxbGKaEFBQIhVs1BrbVCoYrPUGI40OBqpS3BgF9lwUjdg5be4fSpbgAbN6lmQ2Jw5hzC5q1qIuyH3/uYsKtqcFEDqLQa8BadkDjGVt7gxY52EBmfsodOLYW6TiLZmtcnpllt3zKfRULQeUNkDIQVQ9Ff5lSnC/dWRunxDrAWE/T/CKLUlTl81iG04NeTdNFhBjiqVjdUX+Suos14DB3m7/UOlfVaPshiMBuGIXw1mWaer/wCkSLT+T/2Jf936ilV+I/7iREraYdFtsuA2+RGbJMKx8lJYIdJ/YV/UCVpV0n+iYILiy/qU5FqApirNIF6v1dxZbfwGYPzAryVXA85iHAPqGrsbZbeqMsKUJysHNv7I/FtkKAdFZwOIWOYw1Zsbz+IgC2um/lhhRL7yfqGKZ7xXaBmJzVNxbsY+KgZZbSfOFX3AboByDpRcx0HPYk/gIWAGjp9wJXC+oGmdIVbhE/uPyjmUfUb9WRDCBz+3CRAtrtSX6iStHACJ00uQJG30oN/zKAObBH5ghoDQbNAZh0hYGwesRpxTYNn3M8XUvGTdAbhRDqWQ5RfxLD8hS2NZ0IWX0ypT1Yqgdo3KBm0HyWMsIkDDQv7QutMrDgjS9trKAWqfiVhQ0OEdVHLE4pVKutai4IfbcRaHwVMBT9kIKi7Mv43KuOoPkbgk66BXXANRgEnuq/qUdpdmQ/1HgPoCBsd/B+poNfRSMQzT7Vxof3CgoFBxqV1DBEmURG919Ra5zFyNa+O4EC9qA4O+YLAIWyXNPMVlScBr5qcc8llH2wMABLUvYO/cGGRtbVwVnqYQBQ1/lg49ExPtDEHJvqC8nyxGE4ZV9wS4xFo6tbFUaFKj1/b+ojAGFMH1RhzbxQv7shIe6Av4JyvmEsVZAvISkembc1pl36c0Hmqz+5VygUUjd0R6OEhZTwJxHTZzQpPUpWRUKrftCMsCANFcymG0C8uqmp7kBXsgC3pZW4zFwW+kJkYmEfZbK8MpBpD8za0H5LYpgE5HmLL4S6a/E4AHRiLberLAAIU3doNi6JaY16Kl3gMYQQpHqXCTGK7iiHAEfctwAMl1ACDZGZIjAHhP9gmxYd0uZuDgbf8AyJllcAPVzMwCAqjBDDZgm385nymeL8C93FMbMMoyZIXZLu/zBTUZr2mXdxLcTNsaNvzO1Ms51/cA1T5ifvUIfUIUCO6GYMBDWH8SyIsutf4gQfGEPKHVDNpOYIr0gO7gJRge4B5I+k+5R4RBU1OiEBXdSdBaaYgwASymJ0xOmNu0DxLy8HMxgR5IdcC4IhiA9koep6SYdwzbCrCJ8qWgo3cHRiW6i1t8uplil/Gm+EDlhl7+IQriMAIlZgIkN1wwlhiFNqmbEbag5Z+WVoNtRWRiYR/HxADMInphBTljsbtmU1Z/gbzMPSuJWSeADDBlpK9R844ZlatMdyuLdW9S1tSrb3KFEVL9Eq0s0bgUsaYAOAPipUv1LmagX4Lwxu4kjlTQJqPVKbt6jpQ8BuZKUtrtcE6f3BHMwzcvFNF7iaBOiwmzwsOjqWBytSlBIVYSImoGtQTiAMqnDiEA6geoV4hhglzidqIWLEpFPq4I5H7lBiHJntZbuDhMI21AlSVV7uN2K5gwnXtqV7OxsqN3aLINwxATklvqX8RQiHuNdXFDzHOdDEsiibDDMuKdysqyYxKoqwgiWhZDUs7auJaGZbGLNcNRmwMZ4mIAqoKcwvLy3uWlstiyyDpAe40mHDcNKMM4mrBo9Rql+0o0V4q6xLhQY9w1j6eBRspuziNNtwcwblPH35CF9ZnqSnZHWZbiUjAm7j7cIfkQo4s4nLrTcUFojCAm0WJlBumAvA0YCENztcMQS5Y+BCDbCzczZgiXYl6wgbC/MM1MTBZNUS1kgJOBItSqTRheZaluO2c2/Ex/A6gOYM4Z8LlvH4wctYPgKMrrNz0kaSFfBcQMbTjNkVebSsAZEYVpqUXFUIMTOEVEzSZaSS9QXSoEwwdZSWPNSnWYcxGiy1hd7QEtxE6VC8oBhFOZbOXuCXgQz1JRZhEsa8GAimGoqB4BcGhixA8DEQc3Fc1LW7gsweg3Lo024ah5Q0wDmHMZ3IicQl3RmGShHATpwWJEjhZUcytCWLOYRDCktgtnuAFhmYO5vRP/2Q=='; diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts new file mode 100644 index 00000000000..4dea4170c0d --- /dev/null +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -0,0 +1,372 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AIError } from '../errors'; +import { logger } from '../logger'; +import { + CountTokensRequest, + GenerateContentRequest, + InferenceMode, + Part, + AIErrorCode, + OnDeviceParams, + Content, + Role +} from '../types'; +import { ChromeAdapter } from '../types/chrome-adapter'; +import { + Availability, + LanguageModel, + LanguageModelMessage, + LanguageModelMessageContent, + LanguageModelMessageRole +} from '../types/language-model'; + +/** + * Defines an inference "backend" that uses Chrome's on-device model, + * and encapsulates logic for detecting when on-device inference is + * possible. + */ +export class ChromeAdapterImpl implements ChromeAdapter { + // Visible for testing + static SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png']; + private isDownloading = false; + private downloadPromise: Promise | undefined; + private oldSession: LanguageModel | undefined; + constructor( + private languageModelProvider: LanguageModel, + private mode: InferenceMode, + private onDeviceParams: OnDeviceParams = { + createOptions: { + // Defaults to support image inputs for convenience. + expectedInputs: [{ type: 'image' }] + } + } + ) {} + + /** + * Checks if a given request can be made on-device. + * + *
    Encapsulates a few concerns: + *
  1. the mode
    2. + *
  2. API existence
    3. + *
  3. prompt formatting
    4. + *
  4. model availability, including triggering download if necessary
    5. + *
+ * + *

Pros: callers needn't be concerned with details of on-device availability.

+ *

Cons: this method spans a few concerns and splits request validation from usage. + * If instance variables weren't already part of the API, we could consider a better + * separation of concerns.

+ */ + async isAvailable(request: GenerateContentRequest): Promise { + if (!this.mode) { + logger.debug( + `On-device inference unavailable because mode is undefined.` + ); + return false; + } + if (this.mode === InferenceMode.ONLY_IN_CLOUD) { + logger.debug( + `On-device inference unavailable because mode is "only_in_cloud".` + ); + return false; + } + + // Triggers out-of-band download so model will eventually become available. + const availability = await this.downloadIfAvailable(); + + if (this.mode === InferenceMode.ONLY_ON_DEVICE) { + // If it will never be available due to API inavailability, throw. + if (availability === Availability.UNAVAILABLE) { + throw new AIError( + AIErrorCode.API_NOT_ENABLED, + 'Local LanguageModel API not available in this environment.' + ); + } else if ( + availability === Availability.DOWNLOADABLE || + availability === Availability.DOWNLOADING + ) { + // TODO(chholland): Better user experience during download - progress? + logger.debug(`Waiting for download of LanguageModel to complete.`); + await this.downloadPromise; + return true; + } + return true; + } + + // Applies prefer_on_device logic. + if (availability !== Availability.AVAILABLE) { + logger.debug( + `On-device inference unavailable because availability is "${availability}".` + ); + return false; + } + if (!ChromeAdapterImpl.isOnDeviceRequest(request)) { + logger.debug( + `On-device inference unavailable because request is incompatible.` + ); + return false; + } + + return true; + } + + /** + * Generates content on device. + * + *

This is comparable to {@link GenerativeModel.generateContent} for generating content in + * Cloud.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ + async generateContent(request: GenerateContentRequest): Promise { + const session = await this.createSession(); + const contents = await Promise.all( + request.contents.map(ChromeAdapterImpl.toLanguageModelMessage) + ); + const text = await session.prompt( + contents, + this.onDeviceParams.promptOptions + ); + return ChromeAdapterImpl.toResponse(text); + } + + /** + * Generates content stream on device. + * + *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in + * Cloud.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ + async generateContentStream( + request: GenerateContentRequest + ): Promise { + const session = await this.createSession(); + const contents = await Promise.all( + request.contents.map(ChromeAdapterImpl.toLanguageModelMessage) + ); + const stream = session.promptStreaming( + contents, + this.onDeviceParams.promptOptions + ); + return ChromeAdapterImpl.toStreamResponse(stream); + } + + async countTokens(_request: CountTokensRequest): Promise { + throw new AIError( + AIErrorCode.REQUEST_ERROR, + 'Count Tokens is not yet available for on-device model.' + ); + } + + /** + * Asserts inference for the given request can be performed by an on-device model. + */ + private static isOnDeviceRequest(request: GenerateContentRequest): boolean { + // Returns false if the prompt is empty. + if (request.contents.length === 0) { + logger.debug('Empty prompt rejected for on-device inference.'); + return false; + } + + for (const content of request.contents) { + if (content.role === 'function') { + logger.debug(`"Function" role rejected for on-device inference.`); + return false; + } + + // Returns false if request contains an image with an unsupported mime type. + for (const part of content.parts) { + if ( + part.inlineData && + ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf( + part.inlineData.mimeType + ) === -1 + ) { + logger.debug( + `Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.` + ); + return false; + } + } + } + + return true; + } + + /** + * Encapsulates logic to get availability and download a model if one is downloadable. + */ + private async downloadIfAvailable(): Promise { + const availability = await this.languageModelProvider?.availability( + this.onDeviceParams.createOptions + ); + + if (availability === Availability.DOWNLOADABLE) { + this.download(); + } + + return availability; + } + + /** + * Triggers out-of-band download of an on-device model. + * + *

Chrome only downloads models as needed. Chrome knows a model is needed when code calls + * LanguageModel.create.

+ * + *

Since Chrome manages the download, the SDK can only avoid redundant download requests by + * tracking if a download has previously been requested.

+ */ + private download(): void { + if (this.isDownloading) { + return; + } + this.isDownloading = true; + this.downloadPromise = this.languageModelProvider + ?.create(this.onDeviceParams.createOptions) + .finally(() => { + this.isDownloading = false; + }); + } + + /** + * Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object. + */ + private static async toLanguageModelMessage( + content: Content + ): Promise { + const languageModelMessageContents = await Promise.all( + content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent) + ); + return { + role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role), + content: languageModelMessageContents + }; + } + + /** + * Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object. + */ + private static async toLanguageModelMessageContent( + part: Part + ): Promise { + if (part.text) { + return { + type: 'text', + value: part.text + }; + } else if (part.inlineData) { + const formattedImageContent = await fetch( + `data:${part.inlineData.mimeType};base64,${part.inlineData.data}` + ); + const imageBlob = await formattedImageContent.blob(); + const imageBitmap = await createImageBitmap(imageBlob); + return { + type: 'image', + value: imageBitmap + }; + } + throw new AIError( + AIErrorCode.REQUEST_ERROR, + `Processing of this Part type is not currently supported.` + ); + } + + /** + * Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string. + */ + private static toLanguageModelMessageRole( + role: Role + ): LanguageModelMessageRole { + // Assumes 'function' rule has been filtered by isOnDeviceRequest + return role === 'model' ? 'assistant' : 'user'; + } + + /** + * Abstracts Chrome session creation. + * + *

Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all + * inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all + * inference.

+ * + *

Chrome will remove a model from memory if it's no longer in use, so this method ensures a + * new session is created before an old session is destroyed.

+ */ + private async createSession(): Promise { + if (!this.languageModelProvider) { + throw new AIError( + AIErrorCode.UNSUPPORTED, + 'Chrome AI requested for unsupported browser version.' + ); + } + const newSession = await this.languageModelProvider.create( + this.onDeviceParams.createOptions + ); + if (this.oldSession) { + this.oldSession.destroy(); + } + // Holds session reference, so model isn't unloaded from memory. + this.oldSession = newSession; + return newSession; + } + + /** + * Formats string returned by Chrome as a {@link Response} returned by Firebase AI. + */ + private static toResponse(text: string): Response { + return { + json: async () => ({ + candidates: [ + { + content: { + parts: [{ text }] + } + } + ] + }) + } as Response; + } + + /** + * Formats string stream returned by Chrome as SSE returned by Firebase AI. + */ + private static toStreamResponse(stream: ReadableStream): Response { + const encoder = new TextEncoder(); + return { + body: stream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + const json = JSON.stringify({ + candidates: [ + { + content: { + role: 'model', + parts: [{ text: chunk }] + } + } + ] + }); + controller.enqueue(encoder.encode(`data: ${json}\n\n`)); + } + }) + ) + } as Response; + } +} diff --git a/packages/ai/src/methods/count-tokens.test.ts b/packages/ai/src/methods/count-tokens.test.ts index 7e04ddb3561..56985b4d54e 100644 --- a/packages/ai/src/methods/count-tokens.test.ts +++ b/packages/ai/src/methods/count-tokens.test.ts @@ -22,11 +22,12 @@ import chaiAsPromised from 'chai-as-promised'; import { getMockResponse } from '../../test-utils/mock-response'; import * as request from '../requests/request'; import { countTokens } from './count-tokens'; -import { CountTokensRequest } from '../types'; +import { CountTokensRequest, InferenceMode } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; import { mapCountTokensRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -51,6 +52,12 @@ const fakeRequestParams: CountTokensRequest = { contents: [{ parts: [{ text: 'hello' }], role: 'user' }] }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('countTokens()', () => { afterEach(() => { restore(); @@ -66,7 +73,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + fakeChromeAdapter ); expect(result.totalTokens).to.equal(6); expect(result.totalBillableCharacters).to.equal(16); @@ -92,7 +100,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + fakeChromeAdapter ); expect(result.totalTokens).to.equal(1837); expect(result.totalBillableCharacters).to.equal(117); @@ -120,7 +129,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + fakeChromeAdapter ); expect(result.totalTokens).to.equal(258); expect(result).to.not.have.property('totalBillableCharacters'); @@ -146,7 +156,12 @@ describe('countTokens()', () => { json: mockResponse.json } as Response); await expect( - countTokens(fakeApiSettings, 'model', fakeRequestParams) + countTokens( + fakeApiSettings, + 'model', + fakeRequestParams, + fakeChromeAdapter + ) ).to.be.rejectedWith(/404.*not found/); expect(mockFetch).to.be.called; }); @@ -164,7 +179,12 @@ describe('countTokens()', () => { it('maps request to GoogleAI format', async () => { makeRequestStub.resolves({ ok: true, json: () => {} } as Response); // Unused - await countTokens(fakeGoogleAIApiSettings, 'model', fakeRequestParams); + await countTokens( + fakeGoogleAIApiSettings, + 'model', + fakeRequestParams, + fakeChromeAdapter + ); expect(makeRequestStub).to.be.calledWith( 'model', @@ -176,4 +196,24 @@ describe('countTokens()', () => { ); }); }); + it('on-device', async () => { + const chromeAdapter = fakeChromeAdapter; + const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-total-tokens.json' + ); + const countTokensStub = stub(chromeAdapter, 'countTokens').resolves( + mockResponse as Response + ); + const result = await countTokens( + fakeApiSettings, + 'model', + fakeRequestParams, + chromeAdapter + ); + expect(result.totalTokens).eq(6); + expect(isAvailableStub).to.be.called; + expect(countTokensStub).to.be.calledWith(fakeRequestParams); + }); }); diff --git a/packages/ai/src/methods/count-tokens.ts b/packages/ai/src/methods/count-tokens.ts index b1e60e3a182..00dde84ab48 100644 --- a/packages/ai/src/methods/count-tokens.ts +++ b/packages/ai/src/methods/count-tokens.ts @@ -24,8 +24,9 @@ import { Task, makeRequest } from '../requests/request'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; +import { ChromeAdapter } from '../types/chrome-adapter'; -export async function countTokens( +export async function countTokensOnCloud( apiSettings: ApiSettings, model: string, params: CountTokensRequest, @@ -48,3 +49,17 @@ export async function countTokens( ); return response.json(); } + +export async function countTokens( + apiSettings: ApiSettings, + model: string, + params: CountTokensRequest, + chromeAdapter?: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { + return (await chromeAdapter.countTokens(params)).json(); + } + + return countTokensOnCloud(apiSettings, model, params, requestOptions); +} diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index a5e4c0d1b57..19c0761949a 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -27,17 +27,25 @@ import { GenerateContentRequest, HarmBlockMethod, HarmBlockThreshold, - HarmCategory + HarmCategory, + InferenceMode } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; import { AIError } from '../api'; import { mapGenerateContentRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + const fakeApiSettings: ApiSettings = { apiKey: 'key', project: 'my-project', @@ -422,4 +430,25 @@ describe('generateContent()', () => { ); }); }); + // TODO: define a similar test for generateContentStream + it('on-device', async () => { + const chromeAdapter = fakeChromeAdapter; + const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-basic-reply-short.json' + ); + const generateContentStub = stub(chromeAdapter, 'generateContent').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams, + chromeAdapter + ); + expect(result.response.text()).to.include('Mountain View, California'); + expect(isAvailableStub).to.be.called; + expect(generateContentStub).to.be.calledWith(fakeRequestParams); + }); }); diff --git a/packages/ai/src/methods/generate-content.ts b/packages/ai/src/methods/generate-content.ts index 5f7902f5954..2c1c383641f 100644 --- a/packages/ai/src/methods/generate-content.ts +++ b/packages/ai/src/methods/generate-content.ts @@ -28,17 +28,18 @@ import { processStream } from '../requests/stream-reader'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; +import { ChromeAdapter } from '../types/chrome-adapter'; -export async function generateContentStream( +async function generateContentStreamOnCloud( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, requestOptions?: RequestOptions -): Promise { +): Promise { if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) { params = GoogleAIMapper.mapGenerateContentRequest(params); } - const response = await makeRequest( + return makeRequest( model, Task.STREAM_GENERATE_CONTENT, apiSettings, @@ -46,19 +47,39 @@ export async function generateContentStream( JSON.stringify(params), requestOptions ); +} + +export async function generateContentStream( + apiSettings: ApiSettings, + model: string, + params: GenerateContentRequest, + chromeAdapter?: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + let response; + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { + response = await chromeAdapter.generateContentStream(params); + } else { + response = await generateContentStreamOnCloud( + apiSettings, + model, + params, + requestOptions + ); + } return processStream(response, apiSettings); // TODO: Map streaming responses } -export async function generateContent( +async function generateContentOnCloud( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, requestOptions?: RequestOptions -): Promise { +): Promise { if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) { params = GoogleAIMapper.mapGenerateContentRequest(params); } - const response = await makeRequest( + return makeRequest( model, Task.GENERATE_CONTENT, apiSettings, @@ -66,6 +87,26 @@ export async function generateContent( JSON.stringify(params), requestOptions ); +} + +export async function generateContent( + apiSettings: ApiSettings, + model: string, + params: GenerateContentRequest, + chromeAdapter?: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + let response; + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { + response = await chromeAdapter.generateContent(params); + } else { + response = await generateContentOnCloud( + apiSettings, + model, + params, + requestOptions + ); + } const generateContentResponse = await processGenerateContentResponse( response, apiSettings diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index d055b82b1be..68f1565b26a 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -16,12 +16,13 @@ */ import { use, expect } from 'chai'; import { GenerativeModel } from './generative-model'; -import { FunctionCallingMode, AI } from '../public-types'; +import { FunctionCallingMode, AI, InferenceMode } from '../public-types'; import * as request from '../requests/request'; import { match, restore, stub } from 'sinon'; import { getMockResponse } from '../../test-utils/mock-response'; import sinonChai from 'sinon-chai'; import { VertexAIBackend } from '../backend'; +import { ChromeAdapterImpl } from '../methods/chrome-adapter'; use(sinonChai); @@ -39,23 +40,36 @@ const fakeAI: AI = { location: 'us-central1' }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('GenerativeModel', () => { it('passes params through to generateContent', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { - functionDeclarations: [ - { - name: 'myfunc', - description: 'mydesc' - } - ] - } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { + functionDeclarations: [ + { + name: 'myfunc', + description: 'mydesc' + } + ] + } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } + }, + {}, + fakeChromeAdapter + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -86,10 +100,15 @@ describe('GenerativeModel', () => { restore(); }); it('passes text-only systemInstruction through to generateContent', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - systemInstruction: 'be friendly' - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + systemInstruction: 'be friendly' + }, + {}, + fakeChromeAdapter + ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( 'vertexAI', @@ -112,21 +131,28 @@ describe('GenerativeModel', () => { restore(); }); it('generateContent overrides model values', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { - functionDeclarations: [ - { - name: 'myfunc', - description: 'mydesc' - } - ] - } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { + functionDeclarations: [ + { + name: 'myfunc', + description: 'mydesc' + } + ] + } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } + }, + {}, + fakeChromeAdapter + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -168,12 +194,17 @@ describe('GenerativeModel', () => { restore(); }); it('passes base model params through to ChatSession when there are no startChatParams', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + generationConfig: { + topK: 1 + } + }, + {}, + fakeChromeAdapter + ); const chatSession = genModel.startChat(); expect(chatSession.params?.generationConfig).to.deep.equal({ topK: 1 @@ -181,12 +212,17 @@ describe('GenerativeModel', () => { restore(); }); it('overrides base model params with startChatParams', () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + generationConfig: { + topK: 1 + } + }, + {}, + fakeChromeAdapter + ); const chatSession = genModel.startChat({ generationConfig: { topK: 2 @@ -197,17 +233,24 @@ describe('GenerativeModel', () => { }); }); it('passes params through to chat.sendMessage', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + generationConfig: { + topK: 1 + } + }, + {}, + fakeChromeAdapter + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -239,10 +282,15 @@ describe('GenerativeModel', () => { restore(); }); it('passes text-only systemInstruction through to chat.sendMessage', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - systemInstruction: 'be friendly' - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + systemInstruction: 'be friendly' + }, + {}, + fakeChromeAdapter + ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( 'vertexAI', @@ -265,17 +313,24 @@ describe('GenerativeModel', () => { restore(); }); it('startChat overrides model values', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, - generationConfig: { - responseMimeType: 'image/jpeg' - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + generationConfig: { + responseMimeType: 'image/jpeg' + } + }, + {}, + fakeChromeAdapter + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -325,7 +380,12 @@ describe('GenerativeModel', () => { restore(); }); it('calls countTokens', async () => { - const genModel = new GenerativeModel(fakeAI, { model: 'my-model' }); + const genModel = new GenerativeModel( + fakeAI, + { model: 'my-model' }, + {}, + fakeChromeAdapter + ); const mockResponse = getMockResponse( 'vertexAI', 'unary-success-total-tokens.json' diff --git a/packages/ai/src/models/generative-model.ts b/packages/ai/src/models/generative-model.ts index b09a9290aa4..ffce645eeb1 100644 --- a/packages/ai/src/models/generative-model.ts +++ b/packages/ai/src/models/generative-model.ts @@ -43,6 +43,7 @@ import { } from '../requests/request-helpers'; import { AI } from '../public-types'; import { AIModel } from './ai-model'; +import { ChromeAdapter } from '../types/chrome-adapter'; /** * Class for generative model APIs. @@ -59,7 +60,8 @@ export class GenerativeModel extends AIModel { constructor( ai: AI, modelParams: ModelParams, - requestOptions?: RequestOptions + requestOptions?: RequestOptions, + private chromeAdapter?: ChromeAdapter ) { super(ai, modelParams.model); this.generationConfig = modelParams.generationConfig || {}; @@ -91,6 +93,7 @@ export class GenerativeModel extends AIModel { systemInstruction: this.systemInstruction, ...formattedParams }, + this.chromeAdapter, this.requestOptions ); } @@ -116,6 +119,7 @@ export class GenerativeModel extends AIModel { systemInstruction: this.systemInstruction, ...formattedParams }, + this.chromeAdapter, this.requestOptions ); } @@ -128,6 +132,7 @@ export class GenerativeModel extends AIModel { return new ChatSession( this._apiSettings, this.model, + this.chromeAdapter, { tools: this.tools, toolConfig: this.toolConfig, @@ -152,6 +157,11 @@ export class GenerativeModel extends AIModel { request: CountTokensRequest | string | Array ): Promise { const formattedParams = formatGenerateContentInput(request); - return countTokens(this._apiSettings, this.model, formattedParams); + return countTokens( + this._apiSettings, + this.model, + formattedParams, + this.chromeAdapter + ); } } diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts new file mode 100644 index 00000000000..77c52eb9391 --- /dev/null +++ b/packages/ai/src/types/chrome-adapter.ts @@ -0,0 +1,59 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CountTokensRequest, GenerateContentRequest } from './requests'; + +/** + * (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, + * and encapsulates logic for detecting when on-device inference is + * possible. + * + * These methods should not be called directly by the user. + * + * @public + */ +export interface ChromeAdapter { + /** + * Checks if the on-device model is capable of handling a given + * request. + * @param request - A potential request to be passed to the model. + */ + isAvailable(request: GenerateContentRequest): Promise; + + /** + * Generates content using on-device inference. + * + *

This is comparable to {@link GenerativeModel.generateContent} for generating + * content using in-cloud inference.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + */ + generateContent(request: GenerateContentRequest): Promise; + + /** + * Generates a content stream using on-device inference. + * + *

This is comparable to {@link GenerativeModel.generateContentStream} for generating + * a content stream using in-cloud inference.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + */ + generateContentStream(request: GenerateContentRequest): Promise; + + /** + * @internal + */ + countTokens(request: CountTokensRequest): Promise; +} diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index b5e4e60ab4f..956be64ba75 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -67,7 +67,7 @@ export const HarmBlockThreshold = { BLOCK_NONE: 'BLOCK_NONE', /** * All content will be allowed. This is the same as `BLOCK_NONE`, but the metadata corresponding - * to the {@link HarmCategory} will not be present in the response. + * to the {@link (HarmCategory:type)} will not be present in the response. */ OFF: 'OFF' } as const; @@ -270,6 +270,9 @@ export const FunctionCallingMode = { NONE: 'NONE' } as const; +/** + * @public + */ export type FunctionCallingMode = (typeof FunctionCallingMode)[keyof typeof FunctionCallingMode]; @@ -335,3 +338,21 @@ export const ResponseModality = { */ export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; + +/** + * (EXPERIMENTAL) + * Determines whether inference happens on-device or in-cloud. + * @public + */ +export const InferenceMode = { + 'PREFER_ON_DEVICE': 'prefer_on_device', + 'ONLY_ON_DEVICE': 'only_on_device', + 'ONLY_IN_CLOUD': 'only_in_cloud' +} as const; + +/** + * (EXPERIMENTAL) + * Determines whether inference happens on-device or in-cloud. + * @public + */ +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; diff --git a/packages/ai/src/types/imagen/responses.ts b/packages/ai/src/types/imagen/responses.ts index 4e4496e6b36..be99ab104bf 100644 --- a/packages/ai/src/types/imagen/responses.ts +++ b/packages/ai/src/types/imagen/responses.ts @@ -37,6 +37,7 @@ export interface ImagenInlineImage { * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. * * This feature is not available yet. + * @beta */ export interface ImagenGCSImage { /** diff --git a/packages/ai/src/types/index.ts b/packages/ai/src/types/index.ts index 01f3e7a701a..2dfe73040ae 100644 --- a/packages/ai/src/types/index.ts +++ b/packages/ai/src/types/index.ts @@ -23,3 +23,15 @@ export * from './error'; export * from './schema'; export * from './imagen'; export * from './googleai'; +export { + LanguageModelCreateOptions, + LanguageModelCreateCoreOptions, + LanguageModelExpected, + LanguageModelMessage, + LanguageModelMessageContent, + LanguageModelMessageContentValue, + LanguageModelMessageRole, + LanguageModelMessageType, + LanguageModelPromptOptions +} from './language-model'; +export * from './chrome-adapter'; diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts new file mode 100644 index 00000000000..4157e6d05e6 --- /dev/null +++ b/packages/ai/src/types/language-model.ts @@ -0,0 +1,143 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * The subset of the Prompt API + * (see {@link https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl } + * required for hybrid functionality. + * + * @internal + */ +export interface LanguageModel extends EventTarget { + create(options?: LanguageModelCreateOptions): Promise; + availability(options?: LanguageModelCreateCoreOptions): Promise; + prompt( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + promptStreaming( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): ReadableStream; + measureInputUsage( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + destroy(): undefined; +} + +/** + * @internal + */ +export enum Availability { + 'UNAVAILABLE' = 'unavailable', + 'DOWNLOADABLE' = 'downloadable', + 'DOWNLOADING' = 'downloading', + 'AVAILABLE' = 'available' +} + +/** + * (EXPERIMENTAL) + * Configures the creation of an on-device language model session. + * @public + */ +export interface LanguageModelCreateCoreOptions { + topK?: number; + temperature?: number; + expectedInputs?: LanguageModelExpected[]; +} + +/** + * (EXPERIMENTAL) + * Configures the creation of an on-device language model session. + * @public + */ +export interface LanguageModelCreateOptions + extends LanguageModelCreateCoreOptions { + signal?: AbortSignal; + initialPrompts?: LanguageModelMessage[]; +} + +/** + * (EXPERIMENTAL) + * Options for an on-device language model prompt. + * @public + */ +export interface LanguageModelPromptOptions { + responseConstraint?: object; + // TODO: Restore AbortSignal once the API is defined. +} + +/** + * (EXPERIMENTAL) + * Options for the expected inputs for an on-device language model. + * @public + */ export interface LanguageModelExpected { + type: LanguageModelMessageType; + languages?: string[]; +} + +/** + * (EXPERIMENTAL) + * An on-device language model prompt. + * @public + */ +export type LanguageModelPrompt = LanguageModelMessage[]; + +/** + * (EXPERIMENTAL) + * An on-device language model message. + * @public + */ +export interface LanguageModelMessage { + role: LanguageModelMessageRole; + content: LanguageModelMessageContent[]; +} + +/** + * (EXPERIMENTAL) + * An on-device language model content object. + * @public + */ +export interface LanguageModelMessageContent { + type: LanguageModelMessageType; + value: LanguageModelMessageContentValue; +} + +/** + * (EXPERIMENTAL) + * Allowable roles for on-device language model usage. + * @public + */ +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; + +/** + * (EXPERIMENTAL) + * Allowable types for on-device language model messages. + * @public + */ +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; + +/** + * (EXPERIMENTAL) + * Content formats that can be provided as on-device message content. + * @public + */ +export type LanguageModelMessageContentValue = + | ImageBitmapSource + | AudioBuffer + | BufferSource + | string; diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index e68f3af161d..b1bde0bc290 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -17,11 +17,16 @@ import { ObjectSchema, TypedSchema } from '../requests/schema-builder'; import { Content, Part } from './content'; +import { + LanguageModelCreateOptions, + LanguageModelPromptOptions +} from './language-model'; import { FunctionCallingMode, HarmBlockMethod, HarmBlockThreshold, HarmCategory, + InferenceMode, ResponseModality } from './enums'; import { ObjectSchemaRequest, SchemaRequest } from './schema'; @@ -271,6 +276,37 @@ export interface FunctionCallingConfig { allowedFunctionNames?: string[]; } +/** + * (EXPERIMENTAL) + * Encapsulates configuration for on-device inference. + * + * @public + */ +export interface OnDeviceParams { + createOptions?: LanguageModelCreateOptions; + promptOptions?: LanguageModelPromptOptions; +} + +/** + * (EXPERIMENTAL) + * Configures hybrid inference. + * @public + */ +export interface HybridParams { + /** + * Specifies on-device or in-cloud inference. Defaults to prefer on-device. + */ + mode: InferenceMode; + /** + * Optional. Specifies advanced params for on-device inference. + */ + onDeviceParams?: OnDeviceParams; + /** + * Optional. Specifies advanced params for in-cloud inference. + */ + inCloudParams?: ModelParams; +} + /** * Configuration for "thinking" behavior of compatible Gemini models. * diff --git a/packages/ai/src/types/schema.ts b/packages/ai/src/types/schema.ts index f8c91168bf2..8068ce62a91 100644 --- a/packages/ai/src/types/schema.ts +++ b/packages/ai/src/types/schema.ts @@ -128,7 +128,7 @@ export interface SchemaInterface extends SchemaShared { } /** - * Interface for JSON parameters in a schema of {@link SchemaType} + * Interface for JSON parameters in a schema of {@link (SchemaType:type)} * "object" when not using the `Schema.object()` helper. * @public */