@ai-sdk/google - npm Package Compare versions

Comparing version 3.0.86 to 3.0.87

CHANGELOG.md

 # @ai-sdk/google
 
+## 3.0.87
+
+### Patch Changes
+
+- fa850e6: feat (video): add first-class `frameImages` and `inputReferences` call options for video generation
+- Updated dependencies [fa850e6]
+  - @ai-sdk/provider@3.0.13
+  - @ai-sdk/provider-utils@4.0.34
+
 ## 3.0.86
@@ -4,0 +13,0 @@

package.json

 {
   "name": "@ai-sdk/google",
-  "version": "3.0.86",
+  "version": "3.0.87",
   "license": "Apache-2.0",
@@ -39,4 +39,4 @@ "sideEffects": false,
   "dependencies": {
-    "@ai-sdk/provider": "3.0.12",
-    "@ai-sdk/provider-utils": "4.0.33"
+    "@ai-sdk/provider": "3.0.13",
+    "@ai-sdk/provider-utils": "4.0.34"
   },
@@ -43,0 +43,0 @@ "devDependencies": {

src/google-generative-ai-video-model.ts

 import {
   AISDKError,
   type Experimental_VideoModelV3,
+  type Experimental_VideoModelV3File,
   type SharedV3Warning,
@@ -54,2 +55,99 @@ } from '@ai-sdk/provider';
 
+function getFirstFrameImage(
+  options: Parameters<Experimental_VideoModelV3['doGenerate']>[0],
+): Experimental_VideoModelV3File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'first_frame')
+    ?.image;
+}
+
+function resolveStartImage(
+  options: Parameters<Experimental_VideoModelV3['doGenerate']>[0],
+): Experimental_VideoModelV3File | undefined {
+  return getFirstFrameImage(options) ?? options.image;
+}
+
+function getLastFrameImage(
+  options: Parameters<Experimental_VideoModelV3['doGenerate']>[0],
+): Experimental_VideoModelV3File | undefined {
+  return options.frameImages?.find(frame => frame.frameType === 'last_frame')
+    ?.image;
+}
+
+function getInputReferences(
+  options: Parameters<Experimental_VideoModelV3['doGenerate']>[0],
+): Array<Experimental_VideoModelV3File> | undefined {
+  if (options.frameImages != null && options.frameImages.length > 0) {
+    return undefined;
+  }
+
+  return options.inputReferences != null && options.inputReferences.length > 0
+    ? options.inputReferences
+    : undefined;
+}
+
+function convertFileToGoogleImage(
+  file: Experimental_VideoModelV3File,
+  warnings: SharedV3Warning[],
+): Record<string, unknown> | undefined {
+  if (file.type === 'url') {
+    if (file.url.startsWith('gs://')) {
+      return {
+        gcsUri: file.url,
+        mimeType: 'image/png',
+      };
+    }
+
+    warnings.push({
+      type: 'unsupported',
+      feature: 'URL-based image input',
+      details:
+        'Google Generative AI video models require base64-encoded images or GCS URIs. URL will be ignored.',
+    });
+    return undefined;
+  }
+
+  const base64Data =
+    typeof file.data === 'string'
+      ? file.data
+      : convertUint8ArrayToBase64(file.data);
+
+  // The Gemini Developer API (generativelanguage.googleapis.com) requires
+  // inline image bytes wrapped as `inlineData`.
+  return {
+    inlineData: {
+      mimeType: file.mediaType || 'image/png',
+      data: base64Data,
+    },
+  };
+}
+
+function convertProviderReferenceImage(
+  refImg: NonNullable<GoogleVideoModelOptions['referenceImages']>[number],
+): Record<string, unknown> {
+  if (refImg.bytesBase64Encoded) {
+    return {
+      inlineData: {
+        mimeType: 'image/png',
+        data: refImg.bytesBase64Encoded,
+      },
+    };
+  }
+
+  if (refImg.gcsUri) {
+    return {
+      gcsUri: refImg.gcsUri,
+    };
+  }
+
+  return refImg;
+}
+
+function convertInputReferenceImage(
+  file: Experimental_VideoModelV3File,
+  warnings: SharedV3Warning[],
+): Record<string, unknown> | undefined {
+  const image = convertFileToGoogleImage(file, warnings);
+  return image != null ? { image, referenceType: 'asset' } : undefined;
+}
+
 export class GoogleGenerativeAIVideoModel implements Experimental_VideoModelV3 {
@@ -91,42 +189,28 @@ readonly specificationVersion = 'v3';
 
-    // Handle image-to-video: convert image to base64
-    if (options.image != null) {
-      if (options.image.type === 'url') {
-        warnings.push({
-          type: 'unsupported',
-          feature: 'URL-based image input',
-          details:
-            'Google Generative AI video models require base64-encoded images. URL will be ignored.',
-        });
-      } else {
-        const base64Data =
-          typeof options.image.data === 'string'
-            ? options.image.data
-            : convertUint8ArrayToBase64(options.image.data);
+    const startImage = resolveStartImage(options);
+    if (startImage != null) {
+      const image = convertFileToGoogleImage(startImage, warnings);
+      if (image != null) {
+        instance.image = image;
+      }
+    }
 
-        instance.image = {
-          inlineData: {
-            mimeType: options.image.mediaType || 'image/png',
-            data: base64Data,
-          },
-        };
+    const lastFrameImage = getLastFrameImage(options);
+    if (lastFrameImage != null) {
+      const lastFrame = convertFileToGoogleImage(lastFrameImage, warnings);
+      if (lastFrame != null) {
+        instance.lastFrame = lastFrame;
       }
     }
 
-    if (googleOptions?.referenceImages != null) {
-      instance.referenceImages = googleOptions.referenceImages.map(refImg => {
-        if (refImg.bytesBase64Encoded) {
-          return {
-            inlineData: {
-              mimeType: 'image/png',
-              data: refImg.bytesBase64Encoded,
-            },
-          };
-        } else if (refImg.gcsUri) {
-          return {
-            gcsUri: refImg.gcsUri,
-          };
-        }
-        return refImg;
+    const inputReferences = getInputReferences(options);
+    if (inputReferences != null) {
+      instance.referenceImages = inputReferences.flatMap(reference => {
+        const converted = convertInputReferenceImage(reference, warnings);
+        return converted != null ? [converted] : [];
       });
+    } else if (googleOptions?.referenceImages != null) {
+      instance.referenceImages = googleOptions.referenceImages.map(refImg =>
+        convertProviderReferenceImage(refImg),
+      );
     }
@@ -133,0 +217,0 @@

New alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

2384915

0.57%

Lines of code

27381

0.64%

Dependency changes

• + Added

  @ai-sdk/provider@3.0.13
  (transitive)

• + Added

  @ai-sdk/provider-utils@4.0.34
  (transitive)

• - Removed

  @ai-sdk/provider@3.0.12
  (transitive)

• - Removed

  @ai-sdk/provider-utils@4.0.33
  (transitive)

• Updated

  @ai-sdk/provider@3.0.13

• Updated

  @ai-sdk/provider-utils@4.0.34