From 3199b11094990478b06031ddf08b299edd279fee Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 7 Jul 2026 18:13:39 +1000 Subject: [PATCH 1/2] fix(ai-gemini): pass HTTPS image URLs through as fileData in createGeminiImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit URL sources in multimodal image-generation prompts now pass through as fileData.fileUri (Gemini fetches them server-side), matching the chat adapter, instead of being fetched locally and inlined as base64 — which double-buffered the image and could OOM on memory-constrained runtimes such as Cloudflare Workers. Veo keeps its fetch: the predict API's Image type only accepts imageBytes or gcsUri, so there is no URI passthrough to align with. Fixes #907 Co-Authored-By: Claude Fable 5 --- .changeset/gemini-image-url-passthrough.md | 5 ++ docs/config.json | 2 +- docs/media/image-generation.md | 6 ++ packages/ai-gemini/src/adapters/image.ts | 72 +++++++------------ packages/ai-gemini/src/adapters/video.ts | 5 ++ .../ai-gemini/tests/image-adapter.test.ts | 72 ++++++++++++------- 6 files changed, 89 insertions(+), 73 deletions(-) create mode 100644 .changeset/gemini-image-url-passthrough.md diff --git a/.changeset/gemini-image-url-passthrough.md b/.changeset/gemini-image-url-passthrough.md new file mode 100644 index 000000000..02a31c395 --- /dev/null +++ b/.changeset/gemini-image-url-passthrough.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-gemini': patch +--- + +fix(ai-gemini): stop fetching arbitrary HTTPS image URLs in `createGeminiImage`. URL sources in multimodal image-generation prompts now pass through as `fileData.fileUri` (Gemini fetches them server-side), matching the chat adapter. This avoids fetch + base64 double-buffering that could OOM on memory-constrained runtimes such as Cloudflare Workers. diff --git a/docs/config.json b/docs/config.json index c7b16c418..99dd073b9 100644 --- a/docs/config.json +++ b/docs/config.json @@ -288,7 +288,7 @@ "label": "Image Generation", "to": "media/image-generation", "addedAt": "2026-04-15", - "updatedAt": "2026-07-01" + "updatedAt": "2026-07-07" }, { "label": "Video Generation", diff --git a/docs/media/image-generation.md b/docs/media/image-generation.md index 8fffa84a2..eaaf76d3f 100644 --- a/docs/media/image-generation.md +++ b/docs/media/image-generation.md @@ -227,6 +227,12 @@ base64 data — pass whichever you have: OpenAI's edit endpoint requires file uploads; the adapter fetches URL sources and converts base64 to a `File` automatically. +Gemini never fetches URL sources locally — they pass through as +`fileData.fileUri` and Gemini retrieves them server-side, so public HTTPS +URLs, [Files API](https://ai.google.dev/gemini-api/docs/files) URIs, and +`gs://` references all work without buffering the image in your runtime's +memory. + ### Role hints via `metadata.role` When a generation has multiple inputs with different roles (mask vs reference diff --git a/packages/ai-gemini/src/adapters/image.ts b/packages/ai-gemini/src/adapters/image.ts index 7fd59bf67..768dab8a9 100644 --- a/packages/ai-gemini/src/adapters/image.ts +++ b/packages/ai-gemini/src/adapters/image.ts @@ -1,6 +1,5 @@ import { resolveMediaPrompt } from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' -import { arrayBufferToBase64 } from '@tanstack/ai-utils' import { createGeminiClient, generateId, @@ -199,7 +198,7 @@ export class GeminiImageAdapter< }), } - const contents = await this.buildContents(resolved, numberOfImages) + const contents = this.buildContents(resolved, numberOfImages) const response = await this.client.models.generateContent({ model, @@ -220,10 +219,10 @@ export class GeminiImageAdapter< * The generateContent API has no numberOfImages parameter, so when more * than one image is requested a trailing instruction is appended. */ - private async buildContents( + private buildContents( resolved: ResolvedMediaPrompt, numberOfImages: number | undefined, - ): Promise> { + ): string | Array { const countInstruction = numberOfImages && numberOfImages > 1 ? `Generate ${numberOfImages} distinct images.` @@ -235,29 +234,25 @@ export class GeminiImageAdapter< : resolved.text } - const parts: Array = await Promise.all( - resolved.parts.map((part) => { - if (part.type === 'text') { - return Promise.resolve({ text: part.content }) - } - if (part.type === 'image') { - return this.imagePartToGeminiPart(part) - } - // Video / audio parts were rejected in generateImages above. - throw new Error( - `gemini: unsupported prompt part type "${part.type}" in image generation.`, - ) - }), - ) + const parts: Array = resolved.parts.map((part) => { + if (part.type === 'text') { + return { text: part.content } + } + if (part.type === 'image') { + return this.imagePartToGeminiPart(part) + } + // Video / audio parts were rejected in generateImages above. + throw new Error( + `gemini: unsupported prompt part type "${part.type}" in image generation.`, + ) + }) if (countInstruction) { parts.push({ text: countInstruction }) } return [{ role: 'user', parts }] } - private async imagePartToGeminiPart( - part: ImagePart, - ): Promise { + private imagePartToGeminiPart(part: ImagePart): Part { if (part.source.type === 'data') { return { inlineData: { @@ -266,34 +261,15 @@ export class GeminiImageAdapter< }, } } - // For URL sources, prefer passing the URL through as `fileData` when it - // looks like a Google Files API URI; otherwise fetch and inline as base64. - if ( - part.source.value.startsWith('gs://') || - /^https?:\/\/generativelanguage\.googleapis\.com\//.test( - part.source.value, - ) - ) { - return { - fileData: { - fileUri: part.source.value, - ...(part.source.mimeType && { mimeType: part.source.mimeType }), - }, - } - } - const response = await fetch(part.source.value) - if (!response.ok) { - throw new Error( - `Failed to fetch image input (${response.status} ${response.statusText}): ${part.source.value}`, - ) - } - const blob = await response.blob() - const buffer = await blob.arrayBuffer() - const base64 = arrayBufferToBase64(buffer) + // URL sources (public HTTPS, Files API URIs, gs://) pass through as + // `fileData` and Gemini fetches them server-side — same as the chat + // adapter. Fetching locally and inlining as base64 double-buffers the + // image and OOMs on memory-constrained runtimes (e.g. Cloudflare + // Workers). return { - inlineData: { - mimeType: part.source.mimeType || blob.type || 'image/png', - data: base64, + fileData: { + fileUri: part.source.value, + mimeType: part.source.mimeType ?? 'image/jpeg', }, } } diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index b6935e503..6e0216ce7 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -55,6 +55,11 @@ function operationErrorMessage(error: Record): string { * Convert a TanStack image prompt part into the genai `Image` shape Veo * accepts: base64 `imageBytes` (data sources, data: URIs, fetched HTTP * URLs) or a `gcsUri` passthrough for Cloud Storage references. + * + * Unlike `generateContent` (chat / native image generation), Veo's predict + * API has no `fileData.fileUri` equivalent — `Image` only accepts + * `imageBytes` or `gcsUri` — so HTTP(S) URLs must still be fetched and + * inlined here. Prefer `gs://` references on memory-constrained runtimes. */ async function imagePartToVeoImage( part: ImagePart, diff --git a/packages/ai-gemini/tests/image-adapter.test.ts b/packages/ai-gemini/tests/image-adapter.test.ts index 6d3d6ce4f..7f07bb04f 100644 --- a/packages/ai-gemini/tests/image-adapter.test.ts +++ b/packages/ai-gemini/tests/image-adapter.test.ts @@ -799,44 +799,68 @@ describe('Gemini Image Adapter', () => { }) }) - it('fetches arbitrary URL sources and inlines them as base64', async () => { + it('passes arbitrary HTTPS URL sources through as fileData without fetching (#907)', async () => { + // Matches the chat adapter: Gemini fetches URL inputs server-side. + // Fetching locally and inlining as base64 OOMs on memory-constrained + // runtimes (e.g. Cloudflare Workers). const { adapter, mockGenerateContent } = mockedNativeAdapter() - // 'hi' → base64 'aGk=' - const fetchMock = vi.fn().mockResolvedValue( - new Response(new Uint8Array([104, 105]), { - headers: { 'content-type': 'image/jpeg' }, - }), - ) - vi.stubGlobal('fetch', fetchMock) - try { - await generateImage({ - adapter, - prompt: [ - { type: 'text', content: 'Edit this' }, - { - type: 'image', - source: { type: 'url', value: 'https://example.com/photo.jpg' }, - }, - ], - }) - } finally { - vi.unstubAllGlobals() - } + await generateImage({ + adapter, + prompt: [ + { type: 'text', content: 'Edit this' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + }) - expect(fetchMock).toHaveBeenCalledWith('https://example.com/photo.jpg') const args = mockGenerateContent.mock.calls[0]![0] expect(args.contents).toEqual([ { role: 'user', parts: [ { text: 'Edit this' }, - { inlineData: { mimeType: 'image/jpeg', data: 'aGk=' } }, + { + fileData: { + fileUri: 'https://example.com/photo.jpg', + // No source mimeType → same image/jpeg default as chat. + mimeType: 'image/jpeg', + }, + }, ], }, ]) }) + it('uses the provided mimeType for URL sources', async () => { + const { adapter, mockGenerateContent } = mockedNativeAdapter() + + await generateImage({ + adapter, + prompt: [ + { type: 'text', content: 'Edit this' }, + { + type: 'image', + source: { + type: 'url', + value: 'https://example.com/photo.png', + mimeType: 'image/png', + }, + }, + ], + }) + + const args = mockGenerateContent.mock.calls[0]![0] + expect(args.contents[0].parts[1]).toEqual({ + fileData: { + fileUri: 'https://example.com/photo.png', + mimeType: 'image/png', + }, + }) + }) + it('rejects image prompt parts for Imagen models', async () => { const adapter = createGeminiImage( 'imagen-4.0-generate-001', From d3661bba09cd3b0870ada6eb04f1bf5bedaf590d Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:39:59 +1000 Subject: [PATCH 2/2] fix(ai-gemini, ai-openai): error on HTTP(S) URL image inputs that require upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Veo (createGeminiVideo), OpenAI image edits (createOpenaiImage), and OpenAI Sora input_reference (createOpenaiVideo) have no URL passthrough — the provider only accepts uploaded bytes (or gs:// for Veo). Previously an HTTP(S) URL image input was silently fetched and buffered in memory, which can OOM constrained runtimes (e.g. Cloudflare Workers). These paths now throw by default on an HTTP(S) URL image input, with an error pointing to the alternatives. data: URIs (and gs:// for Veo) still work with no flag. Opt back into fetching + buffering via allowUrlFetch: true on the adapter config. Threaded through GeminiVideoConfig, OpenAIImageConfig, and OpenAIVideoConfig. Co-Authored-By: Claude Fable 5 --- ...emini-openai-url-input-error-by-default.md | 18 ++++ docs/media/image-generation.md | 28 ++++-- packages/ai-gemini/src/adapters/video.ts | 36 ++++++-- .../ai-gemini/tests/video-adapter.test.ts | 70 ++++++++++++++- packages/ai-openai/src/adapters/image.ts | 23 ++++- packages/ai-openai/src/adapters/video.ts | 16 +++- .../src/image/image-input-to-file.ts | 21 ++++- .../ai-openai/tests/image-adapter.test.ts | 88 +++++++++++++++++++ .../ai-openai/tests/video-adapter.test.ts | 54 ++++++++++++ .../skills/ai-core/media-generation/SKILL.md | 11 +++ 10 files changed, 344 insertions(+), 21 deletions(-) create mode 100644 .changeset/gemini-openai-url-input-error-by-default.md diff --git a/.changeset/gemini-openai-url-input-error-by-default.md b/.changeset/gemini-openai-url-input-error-by-default.md new file mode 100644 index 000000000..ee65c83da --- /dev/null +++ b/.changeset/gemini-openai-url-input-error-by-default.md @@ -0,0 +1,18 @@ +--- +'@tanstack/ai-gemini': minor +'@tanstack/ai-openai': minor +--- + +fix(ai-gemini, ai-openai): don't buffer arbitrary HTTP(S) URL image inputs by default on paths that require uploaded bytes. + +Gemini **Veo** (`createGeminiVideo`), OpenAI image **edits** (`createOpenaiImage`), and OpenAI **Sora** `input_reference` (`createOpenaiVideo`) have no URL passthrough — the provider only accepts inline bytes (or, for Veo, a `gs://` reference). Previously an HTTP(S) URL image input was silently fetched and buffered in memory, which can OOM memory-constrained runtimes (e.g. Cloudflare Workers). + +These paths now **throw** on an HTTP(S) URL image input by default, with an error pointing to the alternatives. `data:` URIs (and `gs://` for Veo) still work without any flag. To opt back into fetching + buffering, set `allowUrlFetch: true` on the adapter config: + +```ts +createOpenaiImage('gpt-image-1', apiKey, { allowUrlFetch: true }) +createOpenaiVideo('sora-2', apiKey, { allowUrlFetch: true }) +createGeminiVideo('veo-3.1-generate-preview', apiKey, { allowUrlFetch: true }) +``` + +Migration: if you passed HTTP(S) URL image inputs to these adapters, either fetch the bytes yourself and pass a `data:` URI, pass a `gs://` reference (Veo), or set `allowUrlFetch: true`. diff --git a/docs/media/image-generation.md b/docs/media/image-generation.md index eaaf76d3f..9f20e59ab 100644 --- a/docs/media/image-generation.md +++ b/docs/media/image-generation.md @@ -224,15 +224,31 @@ base64 data — pass whichever you have: { type: 'image', source: { type: 'data', value: base64String, mimeType: 'image/png' } } ``` -OpenAI's edit endpoint requires file uploads; the adapter fetches URL sources -and converts base64 to a `File` automatically. - -Gemini never fetches URL sources locally — they pass through as -`fileData.fileUri` and Gemini retrieves them server-side, so public HTTPS -URLs, [Files API](https://ai.google.dev/gemini-api/docs/files) URIs, and +Gemini's native image generation never fetches URL sources locally — they pass +through as `fileData.fileUri` and Gemini retrieves them server-side, so public +HTTPS URLs, [Files API](https://ai.google.dev/gemini-api/docs/files) URIs, and `gs://` references all work without buffering the image in your runtime's memory. +Two paths have no URL passthrough and must upload real bytes — OpenAI's +`/images/edits` (and Sora `input_reference`), and Gemini **Veo** (its predict +API accepts only inline bytes or a `gs://` reference). For these, an HTTP(S) +URL input would have to be downloaded and buffered in memory, which can OOM +memory-constrained runtimes (e.g. Cloudflare Workers). So by default they +**throw** on an HTTP(S) URL image input rather than fetch it. Pass a `data:` +URI (or a `gs://` reference for Veo), or opt into fetching with `allowUrlFetch`: + +```typescript ignore +import { createOpenaiImage } from '@tanstack/ai-openai/adapters' + +// Opt into downloading + buffering HTTP(S) URL image inputs (server runtimes +// with headroom). data: URIs always work without this flag. +const adapter = createOpenaiImage('gpt-image-1', apiKey, { allowUrlFetch: true }) +``` + +The same `allowUrlFetch` option exists on `createOpenaiVideo` and +`createGeminiVideo`. + ### Role hints via `metadata.role` When a generation has multiple inputs with different roles (mask vs reference diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index 6e0216ce7..f5d34201b 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -38,7 +38,17 @@ import type { GeminiClientConfig } from '../utils' * * @experimental Video generation is an experimental feature and may change. */ -export interface GeminiVideoConfig extends GeminiClientConfig {} +export interface GeminiVideoConfig extends GeminiClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs. Veo's predict API accepts + * only inline `imageBytes` or a `gcsUri`, so an HTTP(S) URL has to be + * downloaded and base64-encoded locally — which buffers the whole image in + * memory and can OOM constrained runtimes (e.g. Cloudflare Workers). When + * `false` (the default), HTTP(S) URL image inputs throw; pass a `data:` URI + * or a `gs://` reference, or set this to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * Extract a human-readable message from a long-running operation's error, @@ -58,11 +68,14 @@ function operationErrorMessage(error: Record): string { * * Unlike `generateContent` (chat / native image generation), Veo's predict * API has no `fileData.fileUri` equivalent — `Image` only accepts - * `imageBytes` or `gcsUri` — so HTTP(S) URLs must still be fetched and - * inlined here. Prefer `gs://` references on memory-constrained runtimes. + * `imageBytes` or `gcsUri`. An HTTP(S) URL therefore has to be fetched and + * inlined locally, which buffers the whole image in memory; that only happens + * when the caller opts in via `allowUrlFetch`, otherwise it throws. Prefer a + * `gs://` reference on memory-constrained runtimes. */ async function imagePartToVeoImage( part: ImagePart, + allowUrlFetch: boolean, ): Promise { if (part.source.type === 'data') { return { @@ -89,6 +102,15 @@ async function imagePartToVeoImage( mimeType: match[1] || part.source.mimeType || 'image/png', } } + if (!allowUrlFetch) { + throw new Error( + `gemini Veo: HTTP(S) URL image inputs are not fetched by default because ` + + `Veo accepts only inline bytes, so the image would be downloaded and ` + + `buffered in memory (risking OOM on constrained runtimes). Pass a ` + + `data: URI or a gs:// reference, or set \`allowUrlFetch: true\` on the ` + + `adapter config to opt into fetching. URL: ${url}`, + ) + } const response = await fetch(url) if (!response.ok) { throw new Error( @@ -137,10 +159,12 @@ export class GeminiVideoAdapter< readonly name = 'gemini' as const protected client: GoogleGenAI + private readonly allowUrlFetch: boolean constructor(config: GeminiVideoConfig, model: TModel) { super({}, model) this.client = createGeminiClient(config) + this.allowUrlFetch = config.allowUrlFetch ?? false } async createVideoJob( @@ -229,13 +253,13 @@ export class GeminiVideoAdapter< `${this.name}: Veo accepts at most one 'end_frame' image.`, ) } - lastFrame = await imagePartToVeoImage(part) + lastFrame = await imagePartToVeoImage(part, this.allowUrlFetch) break } case 'reference': case 'character': { referenceImages.push({ - image: await imagePartToVeoImage(part), + image: await imagePartToVeoImage(part, this.allowUrlFetch), referenceType: VideoGenerationReferenceType.ASSET, }) break @@ -247,7 +271,7 @@ export class GeminiVideoAdapter< `${this.name}: Veo accepts at most one starting image; received multiple 'start_frame'/un-roled images. Use metadata.role ('end_frame', 'reference') to disambiguate the others.`, ) } - image = await imagePartToVeoImage(part) + image = await imagePartToVeoImage(part, this.allowUrlFetch) break } case 'mask': diff --git a/packages/ai-gemini/tests/video-adapter.test.ts b/packages/ai-gemini/tests/video-adapter.test.ts index 5763d6737..1b8202dac 100644 --- a/packages/ai-gemini/tests/video-adapter.test.ts +++ b/packages/ai-gemini/tests/video-adapter.test.ts @@ -56,8 +56,12 @@ function createClientStub( class StubbedGeminiVideoAdapter< TModel extends GeminiVideoModel, > extends GeminiVideoAdapter { - constructor(model: TModel, stub: ClientStub) { - super({ apiKey: 'test-key' }, model) + constructor( + model: TModel, + stub: ClientStub, + config?: { allowUrlFetch?: boolean }, + ) { + super({ apiKey: 'test-key', ...config }, model) this.client = stub as unknown as GoogleGenAI } } @@ -300,6 +304,68 @@ describe('Gemini Video Adapter', () => { }) }) + it('throws on an HTTP(S) URL image input by default instead of buffering it (#907)', async () => { + const stub = createClientStub() + const adapter = new StubbedGeminiVideoAdapter( + 'veo-3.1-generate-preview', + stub, + ) + + await expect( + adapter.createVideoJob({ + model: 'veo-3.1-generate-preview', + prompt: [ + { type: 'text', content: 'animate' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/frame.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(stub.models.generateVideos).not.toHaveBeenCalled() + }) + + it('fetches HTTP(S) URL image inputs when allowUrlFetch is set', async () => { + const stub = createClientStub() + const adapter = new StubbedGeminiVideoAdapter( + 'veo-3.1-generate-preview', + stub, + { allowUrlFetch: true }, + ) + // 'hi' → base64 'aGk=' + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + await adapter.createVideoJob({ + model: 'veo-3.1-generate-preview', + prompt: [ + { type: 'text', content: 'animate' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/frame.jpg' }, + }, + ], + logger: testLogger, + }) + } finally { + vi.unstubAllGlobals() + } + + expect(fetchMock).toHaveBeenCalledWith('https://example.com/frame.jpg') + const call = stub.models.generateVideos.mock.calls[0]?.[0] + expect(call.image).toEqual({ + imageBytes: 'aGk=', + mimeType: 'image/jpeg', + }) + }) + it('rejects multiple starting images', async () => { const stub = createClientStub() const adapter = new StubbedGeminiVideoAdapter( diff --git a/packages/ai-openai/src/adapters/image.ts b/packages/ai-openai/src/adapters/image.ts index d11a67b67..53e9346e1 100644 --- a/packages/ai-openai/src/adapters/image.ts +++ b/packages/ai-openai/src/adapters/image.ts @@ -41,7 +41,17 @@ const EDIT_MAX_IMAGES: Record = { /** * Configuration for OpenAI image adapter */ -export interface OpenAIImageConfig extends OpenAIClientConfig {} +export interface OpenAIImageConfig extends OpenAIClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs for image edits. OpenAI's + * `/images/edits` endpoint requires uploaded file bytes (no URL + * passthrough), so an HTTP(S) URL has to be downloaded and buffered in + * memory — which can OOM constrained runtimes (e.g. Cloudflare Workers). + * When `false` (the default), HTTP(S) URL image inputs throw; pass a `data:` + * URI, or set this to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * OpenAI Image Generation Adapter @@ -67,10 +77,13 @@ export class OpenAIImageAdapter< readonly name = 'openai' as const protected client: OpenAI + private readonly allowUrlFetch: boolean constructor(config: OpenAIImageConfig, model: TModel) { super(model, {}) - this.client = new OpenAI(config) + const { allowUrlFetch, ...clientOptions } = config + this.client = new OpenAI(clientOptions) + this.allowUrlFetch = allowUrlFetch ?? false } async generateImages( @@ -226,11 +239,13 @@ export class OpenAIImageAdapter< } const sourceFiles = await Promise.all( - sourceParts.map((part, i) => imagePartToFile(part, `source-${i}`)), + sourceParts.map((part, i) => + imagePartToFile(part, `source-${i}`, this.allowUrlFetch), + ), ) const [firstSourceFile] = sourceFiles const maskFile = maskParts[0] - ? await imagePartToFile(maskParts[0], 'mask') + ? await imagePartToFile(maskParts[0], 'mask', this.allowUrlFetch) : undefined // `modelOptions` is typed across all four image models (including dall-e-3's diff --git a/packages/ai-openai/src/adapters/video.ts b/packages/ai-openai/src/adapters/video.ts index e28a34be5..3e76478d6 100644 --- a/packages/ai-openai/src/adapters/video.ts +++ b/packages/ai-openai/src/adapters/video.ts @@ -48,7 +48,17 @@ function warnIfLargeMediaBuffer(byteLength: number, source: string): void { * * @experimental Video generation is an experimental feature and may change. */ -export interface OpenAIVideoConfig extends OpenAIClientConfig {} +export interface OpenAIVideoConfig extends OpenAIClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs for Sora's `input_reference`. + * The endpoint requires uploaded file bytes (no URL passthrough), so an + * HTTP(S) URL has to be downloaded and buffered in memory — which can OOM + * constrained runtimes (e.g. Cloudflare Workers). When `false` (the + * default), HTTP(S) URL image inputs throw; pass a `data:` URI, or set this + * to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * OpenAI Video Generation Adapter @@ -84,7 +94,8 @@ export class OpenAIVideoAdapter< // We hold our own typed copy on `clientConfig` and pass an empty object up. super({}, model) this.clientConfig = config - this.client = new OpenAI(config) + const { allowUrlFetch: _allowUrlFetch, ...clientOptions } = config + this.client = new OpenAI(clientOptions) } async createVideoJob( @@ -126,6 +137,7 @@ export class OpenAIVideoAdapter< request.input_reference = await imagePartToFile( inputReference, 'input-reference', + this.clientConfig.allowUrlFetch ?? false, ) } // `VideoCreateParams.size` is `size?: VideoSize` (no `| undefined`), so we diff --git a/packages/ai-openai/src/image/image-input-to-file.ts b/packages/ai-openai/src/image/image-input-to-file.ts index 2074496fd..77c3f2a30 100644 --- a/packages/ai-openai/src/image/image-input-to-file.ts +++ b/packages/ai-openai/src/image/image-input-to-file.ts @@ -27,7 +27,12 @@ function ensureFileSupport(): void { * Convert a TanStack `ImagePart` into an OpenAI-compatible `File`. * * - `source.type === 'data'`: decode base64 → Buffer → File. - * - `source.type === 'url'`: fetch the URL (or parse data: URI) → File. + * - `source.type === 'url'` with a `data:` URI: parse in-memory → File. + * - `source.type === 'url'` with an HTTP(S) URL: fetch → File, but only when + * `allowUrlFetch` is set. OpenAI's `/images/edits` and Sora + * `input_reference` require real file bytes (no URL passthrough), so the + * image has to be downloaded and buffered in memory — which can OOM + * constrained runtimes. Off by default; the caller opts in. * * The mime type comes from the source when available, else inferred from the * URL extension, else `image/png`. @@ -35,6 +40,7 @@ function ensureFileSupport(): void { export async function imagePartToFile( part: ImagePart, fallbackName: string, + allowUrlFetch: boolean, ): Promise { ensureFileSupport() @@ -46,6 +52,19 @@ export async function imagePartToFile( }) } + // Remote HTTP(S) URLs must be downloaded and buffered before upload; gate + // that behind an explicit opt-in. `data:` URIs are already in memory, so + // they're handled uniformly via fetch() below without the flag. + if (/^https?:\/\//i.test(part.source.value) && !allowUrlFetch) { + throw new Error( + `openai: HTTP(S) URL image inputs are not fetched by default because ` + + `OpenAI's edit / input_reference endpoints require uploaded bytes, so ` + + `the image would be downloaded and buffered in memory (risking OOM on ` + + `constrained runtimes). Pass a data: URI, or set \`allowUrlFetch: true\` ` + + `on the adapter config to opt into fetching. URL: ${part.source.value}`, + ) + } + // URL source — also handles data: URIs uniformly via fetch(). const response = await fetch(part.source.value) if (!response.ok) { diff --git a/packages/ai-openai/tests/image-adapter.test.ts b/packages/ai-openai/tests/image-adapter.test.ts index ffae82297..010721080 100644 --- a/packages/ai-openai/tests/image-adapter.test.ts +++ b/packages/ai-openai/tests/image-adapter.test.ts @@ -297,6 +297,94 @@ describe('OpenAI Image Adapter', () => { expect(result.images[0]!.b64Json).toBe('edited-base64') }) + it('throws on an HTTP(S) URL image input by default instead of buffering it (#907)', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key' }, + 'gpt-image-1', + ) + const editSpy = adapter.spyOnImagesEdit() + + await expect( + adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(editSpy).not.toHaveBeenCalled() + }) + + it('fetches HTTP(S) URL image inputs when allowUrlFetch is set', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key', allowUrlFetch: true }, + 'gpt-image-1', + ) + const editSpy = adapter + .spyOnImagesEdit() + .mockResolvedValueOnce(imagesEditResponse) + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + logger: testLogger, + }) + expect(fetchMock).toHaveBeenCalledWith('https://example.com/photo.jpg') + expect(editSpy).toHaveBeenCalledTimes(1) + expect(editSpy.mock.calls[0]![0].image).toBeInstanceOf(File) + expect(result.images[0]!.b64Json).toBe('edited-base64') + } finally { + vi.unstubAllGlobals() + } + }) + + it('accepts a data: URI URL source without allowUrlFetch (in-memory, no network)', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key' }, + 'gpt-image-1', + ) + const editSpy = adapter + .spyOnImagesEdit() + .mockResolvedValueOnce(imagesEditResponse) + + const result = await adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { + type: 'url', + value: 'data:image/png;base64,aGVsbG8=', + }, + }, + ], + logger: testLogger, + }) + + expect(editSpy).toHaveBeenCalledTimes(1) + expect(editSpy.mock.calls[0]![0].image).toBeInstanceOf(File) + expect(result.images[0]!.b64Json).toBe('edited-base64') + }) + it('rejects dall-e-3 with a clear error when the prompt has image parts', async () => { const adapter = new TestOpenAIImageAdapter( { apiKey: 'test-api-key' }, diff --git a/packages/ai-openai/tests/video-adapter.test.ts b/packages/ai-openai/tests/video-adapter.test.ts index 5d28241a8..158b85ce8 100644 --- a/packages/ai-openai/tests/video-adapter.test.ts +++ b/packages/ai-openai/tests/video-adapter.test.ts @@ -51,6 +51,60 @@ describe('OpenAI Video Adapter', () => { expect(result.model).toBe('sora-2') }) + it('throws on an HTTP(S) URL input_reference by default instead of buffering it (#907)', async () => { + const { adapter, mockCreate } = mockedAdapter() + + await expect( + adapter.createVideoJob({ + model: 'sora-2', + prompt: [ + { type: 'text', content: 'Slow cinematic push-in' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/ref.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('fetches an HTTP(S) URL input_reference when allowUrlFetch is set', async () => { + const adapter = createOpenaiVideo('sora-2', 'test-api-key', { + allowUrlFetch: true, + }) + const mockCreate = vi.fn().mockResolvedValue({ id: 'video-job-2' }) + ;(adapter as unknown as { client: { videos: unknown } }).client = { + videos: { create: mockCreate }, + } + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + await adapter.createVideoJob({ + model: 'sora-2', + prompt: [ + { type: 'text', content: 'Slow cinematic push-in' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/ref.jpg' }, + }, + ], + logger: testLogger, + }) + } finally { + vi.unstubAllGlobals() + } + + expect(fetchMock).toHaveBeenCalledWith('https://example.com/ref.jpg') + expect(mockCreate.mock.calls[0]![0].input_reference).toBeInstanceOf(File) + }) + it('throws when more than one image part is provided', async () => { const { adapter, mockCreate } = mockedAdapter() diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index b55638111..3c5258981 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -261,6 +261,17 @@ await generateVideo({ }) ``` +**URL inputs that require an upload throw by default.** Most adapters pass a +`type: 'url'` source straight through to the provider. Three paths can't — +OpenAI `images.edit()`, OpenAI Sora `input_reference`, and Gemini **Veo** — +because the provider only accepts uploaded bytes (Veo also takes a `gs://` +reference). For those, an HTTP(S) URL would have to be downloaded and buffered +in memory, which can OOM constrained runtimes, so they **throw** on an HTTP(S) +URL image input by default. Pass a `data:` URI (or `gs://` for Veo), or opt in +with `allowUrlFetch: true` on the adapter config +(`createOpenaiImage(model, apiKey, { allowUrlFetch: true })`, and likewise on +`createOpenaiVideo` / `createGeminiVideo`). `data:` URIs never need the flag. + **Role hints** (`metadata.role`): | Role | Maps to |