diff --git a/.changeset/gemini-image-url-passthrough.md b/.changeset/gemini-image-url-passthrough.md new file mode 100644 index 000000000..02a31c395 --- /dev/null +++ b/.changeset/gemini-image-url-passthrough.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-gemini': patch +--- + +fix(ai-gemini): stop fetching arbitrary HTTPS image URLs in `createGeminiImage`. URL sources in multimodal image-generation prompts now pass through as `fileData.fileUri` (Gemini fetches them server-side), matching the chat adapter. This avoids fetch + base64 double-buffering that could OOM on memory-constrained runtimes such as Cloudflare Workers. diff --git a/.changeset/gemini-openai-url-input-error-by-default.md b/.changeset/gemini-openai-url-input-error-by-default.md new file mode 100644 index 000000000..ee65c83da --- /dev/null +++ b/.changeset/gemini-openai-url-input-error-by-default.md @@ -0,0 +1,18 @@ +--- +'@tanstack/ai-gemini': minor +'@tanstack/ai-openai': minor +--- + +fix(ai-gemini, ai-openai): don't buffer arbitrary HTTP(S) URL image inputs by default on paths that require uploaded bytes. + +Gemini **Veo** (`createGeminiVideo`), OpenAI image **edits** (`createOpenaiImage`), and OpenAI **Sora** `input_reference` (`createOpenaiVideo`) have no URL passthrough — the provider only accepts inline bytes (or, for Veo, a `gs://` reference). Previously an HTTP(S) URL image input was silently fetched and buffered in memory, which can OOM memory-constrained runtimes (e.g. Cloudflare Workers). + +These paths now **throw** on an HTTP(S) URL image input by default, with an error pointing to the alternatives. `data:` URIs (and `gs://` for Veo) still work without any flag. To opt back into fetching + buffering, set `allowUrlFetch: true` on the adapter config: + +```ts +createOpenaiImage('gpt-image-1', apiKey, { allowUrlFetch: true }) +createOpenaiVideo('sora-2', apiKey, { allowUrlFetch: true }) +createGeminiVideo('veo-3.1-generate-preview', apiKey, { allowUrlFetch: true }) +``` + +Migration: if you passed HTTP(S) URL image inputs to these adapters, either fetch the bytes yourself and pass a `data:` URI, pass a `gs://` reference (Veo), or set `allowUrlFetch: true`. diff --git a/docs/config.json b/docs/config.json index c7b16c418..99dd073b9 100644 --- a/docs/config.json +++ b/docs/config.json @@ -288,7 +288,7 @@ "label": "Image Generation", "to": "media/image-generation", "addedAt": "2026-04-15", - "updatedAt": "2026-07-01" + "updatedAt": "2026-07-07" }, { "label": "Video Generation", diff --git a/docs/media/image-generation.md b/docs/media/image-generation.md index 8fffa84a2..9f20e59ab 100644 --- a/docs/media/image-generation.md +++ b/docs/media/image-generation.md @@ -224,8 +224,30 @@ base64 data — pass whichever you have: { type: 'image', source: { type: 'data', value: base64String, mimeType: 'image/png' } } ``` -OpenAI's edit endpoint requires file uploads; the adapter fetches URL sources -and converts base64 to a `File` automatically. +Gemini's native image generation never fetches URL sources locally — they pass +through as `fileData.fileUri` and Gemini retrieves them server-side, so public +HTTPS URLs, [Files API](https://ai.google.dev/gemini-api/docs/files) URIs, and +`gs://` references all work without buffering the image in your runtime's +memory. + +Two paths have no URL passthrough and must upload real bytes — OpenAI's +`/images/edits` (and Sora `input_reference`), and Gemini **Veo** (its predict +API accepts only inline bytes or a `gs://` reference). For these, an HTTP(S) +URL input would have to be downloaded and buffered in memory, which can OOM +memory-constrained runtimes (e.g. Cloudflare Workers). So by default they +**throw** on an HTTP(S) URL image input rather than fetch it. Pass a `data:` +URI (or a `gs://` reference for Veo), or opt into fetching with `allowUrlFetch`: + +```typescript ignore +import { createOpenaiImage } from '@tanstack/ai-openai/adapters' + +// Opt into downloading + buffering HTTP(S) URL image inputs (server runtimes +// with headroom). data: URIs always work without this flag. +const adapter = createOpenaiImage('gpt-image-1', apiKey, { allowUrlFetch: true }) +``` + +The same `allowUrlFetch` option exists on `createOpenaiVideo` and +`createGeminiVideo`. ### Role hints via `metadata.role` diff --git a/packages/ai-gemini/src/adapters/image.ts b/packages/ai-gemini/src/adapters/image.ts index 7fd59bf67..768dab8a9 100644 --- a/packages/ai-gemini/src/adapters/image.ts +++ b/packages/ai-gemini/src/adapters/image.ts @@ -1,6 +1,5 @@ import { resolveMediaPrompt } from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' -import { arrayBufferToBase64 } from '@tanstack/ai-utils' import { createGeminiClient, generateId, @@ -199,7 +198,7 @@ export class GeminiImageAdapter< }), } - const contents = await this.buildContents(resolved, numberOfImages) + const contents = this.buildContents(resolved, numberOfImages) const response = await this.client.models.generateContent({ model, @@ -220,10 +219,10 @@ export class GeminiImageAdapter< * The generateContent API has no numberOfImages parameter, so when more * than one image is requested a trailing instruction is appended. */ - private async buildContents( + private buildContents( resolved: ResolvedMediaPrompt, numberOfImages: number | undefined, - ): Promise> { + ): string | Array { const countInstruction = numberOfImages && numberOfImages > 1 ? `Generate ${numberOfImages} distinct images.` @@ -235,29 +234,25 @@ export class GeminiImageAdapter< : resolved.text } - const parts: Array = await Promise.all( - resolved.parts.map((part) => { - if (part.type === 'text') { - return Promise.resolve({ text: part.content }) - } - if (part.type === 'image') { - return this.imagePartToGeminiPart(part) - } - // Video / audio parts were rejected in generateImages above. - throw new Error( - `gemini: unsupported prompt part type "${part.type}" in image generation.`, - ) - }), - ) + const parts: Array = resolved.parts.map((part) => { + if (part.type === 'text') { + return { text: part.content } + } + if (part.type === 'image') { + return this.imagePartToGeminiPart(part) + } + // Video / audio parts were rejected in generateImages above. + throw new Error( + `gemini: unsupported prompt part type "${part.type}" in image generation.`, + ) + }) if (countInstruction) { parts.push({ text: countInstruction }) } return [{ role: 'user', parts }] } - private async imagePartToGeminiPart( - part: ImagePart, - ): Promise { + private imagePartToGeminiPart(part: ImagePart): Part { if (part.source.type === 'data') { return { inlineData: { @@ -266,34 +261,15 @@ export class GeminiImageAdapter< }, } } - // For URL sources, prefer passing the URL through as `fileData` when it - // looks like a Google Files API URI; otherwise fetch and inline as base64. - if ( - part.source.value.startsWith('gs://') || - /^https?:\/\/generativelanguage\.googleapis\.com\//.test( - part.source.value, - ) - ) { - return { - fileData: { - fileUri: part.source.value, - ...(part.source.mimeType && { mimeType: part.source.mimeType }), - }, - } - } - const response = await fetch(part.source.value) - if (!response.ok) { - throw new Error( - `Failed to fetch image input (${response.status} ${response.statusText}): ${part.source.value}`, - ) - } - const blob = await response.blob() - const buffer = await blob.arrayBuffer() - const base64 = arrayBufferToBase64(buffer) + // URL sources (public HTTPS, Files API URIs, gs://) pass through as + // `fileData` and Gemini fetches them server-side — same as the chat + // adapter. Fetching locally and inlining as base64 double-buffers the + // image and OOMs on memory-constrained runtimes (e.g. Cloudflare + // Workers). return { - inlineData: { - mimeType: part.source.mimeType || blob.type || 'image/png', - data: base64, + fileData: { + fileUri: part.source.value, + mimeType: part.source.mimeType ?? 'image/jpeg', }, } } diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index b6935e503..f5d34201b 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -38,7 +38,17 @@ import type { GeminiClientConfig } from '../utils' * * @experimental Video generation is an experimental feature and may change. */ -export interface GeminiVideoConfig extends GeminiClientConfig {} +export interface GeminiVideoConfig extends GeminiClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs. Veo's predict API accepts + * only inline `imageBytes` or a `gcsUri`, so an HTTP(S) URL has to be + * downloaded and base64-encoded locally — which buffers the whole image in + * memory and can OOM constrained runtimes (e.g. Cloudflare Workers). When + * `false` (the default), HTTP(S) URL image inputs throw; pass a `data:` URI + * or a `gs://` reference, or set this to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * Extract a human-readable message from a long-running operation's error, @@ -55,9 +65,17 @@ function operationErrorMessage(error: Record): string { * Convert a TanStack image prompt part into the genai `Image` shape Veo * accepts: base64 `imageBytes` (data sources, data: URIs, fetched HTTP * URLs) or a `gcsUri` passthrough for Cloud Storage references. + * + * Unlike `generateContent` (chat / native image generation), Veo's predict + * API has no `fileData.fileUri` equivalent — `Image` only accepts + * `imageBytes` or `gcsUri`. An HTTP(S) URL therefore has to be fetched and + * inlined locally, which buffers the whole image in memory; that only happens + * when the caller opts in via `allowUrlFetch`, otherwise it throws. Prefer a + * `gs://` reference on memory-constrained runtimes. */ async function imagePartToVeoImage( part: ImagePart, + allowUrlFetch: boolean, ): Promise { if (part.source.type === 'data') { return { @@ -84,6 +102,15 @@ async function imagePartToVeoImage( mimeType: match[1] || part.source.mimeType || 'image/png', } } + if (!allowUrlFetch) { + throw new Error( + `gemini Veo: HTTP(S) URL image inputs are not fetched by default because ` + + `Veo accepts only inline bytes, so the image would be downloaded and ` + + `buffered in memory (risking OOM on constrained runtimes). Pass a ` + + `data: URI or a gs:// reference, or set \`allowUrlFetch: true\` on the ` + + `adapter config to opt into fetching. URL: ${url}`, + ) + } const response = await fetch(url) if (!response.ok) { throw new Error( @@ -132,10 +159,12 @@ export class GeminiVideoAdapter< readonly name = 'gemini' as const protected client: GoogleGenAI + private readonly allowUrlFetch: boolean constructor(config: GeminiVideoConfig, model: TModel) { super({}, model) this.client = createGeminiClient(config) + this.allowUrlFetch = config.allowUrlFetch ?? false } async createVideoJob( @@ -224,13 +253,13 @@ export class GeminiVideoAdapter< `${this.name}: Veo accepts at most one 'end_frame' image.`, ) } - lastFrame = await imagePartToVeoImage(part) + lastFrame = await imagePartToVeoImage(part, this.allowUrlFetch) break } case 'reference': case 'character': { referenceImages.push({ - image: await imagePartToVeoImage(part), + image: await imagePartToVeoImage(part, this.allowUrlFetch), referenceType: VideoGenerationReferenceType.ASSET, }) break @@ -242,7 +271,7 @@ export class GeminiVideoAdapter< `${this.name}: Veo accepts at most one starting image; received multiple 'start_frame'/un-roled images. Use metadata.role ('end_frame', 'reference') to disambiguate the others.`, ) } - image = await imagePartToVeoImage(part) + image = await imagePartToVeoImage(part, this.allowUrlFetch) break } case 'mask': diff --git a/packages/ai-gemini/tests/image-adapter.test.ts b/packages/ai-gemini/tests/image-adapter.test.ts index 6d3d6ce4f..7f07bb04f 100644 --- a/packages/ai-gemini/tests/image-adapter.test.ts +++ b/packages/ai-gemini/tests/image-adapter.test.ts @@ -799,44 +799,68 @@ describe('Gemini Image Adapter', () => { }) }) - it('fetches arbitrary URL sources and inlines them as base64', async () => { + it('passes arbitrary HTTPS URL sources through as fileData without fetching (#907)', async () => { + // Matches the chat adapter: Gemini fetches URL inputs server-side. + // Fetching locally and inlining as base64 OOMs on memory-constrained + // runtimes (e.g. Cloudflare Workers). const { adapter, mockGenerateContent } = mockedNativeAdapter() - // 'hi' → base64 'aGk=' - const fetchMock = vi.fn().mockResolvedValue( - new Response(new Uint8Array([104, 105]), { - headers: { 'content-type': 'image/jpeg' }, - }), - ) - vi.stubGlobal('fetch', fetchMock) - try { - await generateImage({ - adapter, - prompt: [ - { type: 'text', content: 'Edit this' }, - { - type: 'image', - source: { type: 'url', value: 'https://example.com/photo.jpg' }, - }, - ], - }) - } finally { - vi.unstubAllGlobals() - } + await generateImage({ + adapter, + prompt: [ + { type: 'text', content: 'Edit this' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + }) - expect(fetchMock).toHaveBeenCalledWith('https://example.com/photo.jpg') const args = mockGenerateContent.mock.calls[0]![0] expect(args.contents).toEqual([ { role: 'user', parts: [ { text: 'Edit this' }, - { inlineData: { mimeType: 'image/jpeg', data: 'aGk=' } }, + { + fileData: { + fileUri: 'https://example.com/photo.jpg', + // No source mimeType → same image/jpeg default as chat. + mimeType: 'image/jpeg', + }, + }, ], }, ]) }) + it('uses the provided mimeType for URL sources', async () => { + const { adapter, mockGenerateContent } = mockedNativeAdapter() + + await generateImage({ + adapter, + prompt: [ + { type: 'text', content: 'Edit this' }, + { + type: 'image', + source: { + type: 'url', + value: 'https://example.com/photo.png', + mimeType: 'image/png', + }, + }, + ], + }) + + const args = mockGenerateContent.mock.calls[0]![0] + expect(args.contents[0].parts[1]).toEqual({ + fileData: { + fileUri: 'https://example.com/photo.png', + mimeType: 'image/png', + }, + }) + }) + it('rejects image prompt parts for Imagen models', async () => { const adapter = createGeminiImage( 'imagen-4.0-generate-001', diff --git a/packages/ai-gemini/tests/video-adapter.test.ts b/packages/ai-gemini/tests/video-adapter.test.ts index 5763d6737..1b8202dac 100644 --- a/packages/ai-gemini/tests/video-adapter.test.ts +++ b/packages/ai-gemini/tests/video-adapter.test.ts @@ -56,8 +56,12 @@ function createClientStub( class StubbedGeminiVideoAdapter< TModel extends GeminiVideoModel, > extends GeminiVideoAdapter { - constructor(model: TModel, stub: ClientStub) { - super({ apiKey: 'test-key' }, model) + constructor( + model: TModel, + stub: ClientStub, + config?: { allowUrlFetch?: boolean }, + ) { + super({ apiKey: 'test-key', ...config }, model) this.client = stub as unknown as GoogleGenAI } } @@ -300,6 +304,68 @@ describe('Gemini Video Adapter', () => { }) }) + it('throws on an HTTP(S) URL image input by default instead of buffering it (#907)', async () => { + const stub = createClientStub() + const adapter = new StubbedGeminiVideoAdapter( + 'veo-3.1-generate-preview', + stub, + ) + + await expect( + adapter.createVideoJob({ + model: 'veo-3.1-generate-preview', + prompt: [ + { type: 'text', content: 'animate' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/frame.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(stub.models.generateVideos).not.toHaveBeenCalled() + }) + + it('fetches HTTP(S) URL image inputs when allowUrlFetch is set', async () => { + const stub = createClientStub() + const adapter = new StubbedGeminiVideoAdapter( + 'veo-3.1-generate-preview', + stub, + { allowUrlFetch: true }, + ) + // 'hi' → base64 'aGk=' + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + await adapter.createVideoJob({ + model: 'veo-3.1-generate-preview', + prompt: [ + { type: 'text', content: 'animate' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/frame.jpg' }, + }, + ], + logger: testLogger, + }) + } finally { + vi.unstubAllGlobals() + } + + expect(fetchMock).toHaveBeenCalledWith('https://example.com/frame.jpg') + const call = stub.models.generateVideos.mock.calls[0]?.[0] + expect(call.image).toEqual({ + imageBytes: 'aGk=', + mimeType: 'image/jpeg', + }) + }) + it('rejects multiple starting images', async () => { const stub = createClientStub() const adapter = new StubbedGeminiVideoAdapter( diff --git a/packages/ai-openai/src/adapters/image.ts b/packages/ai-openai/src/adapters/image.ts index d11a67b67..53e9346e1 100644 --- a/packages/ai-openai/src/adapters/image.ts +++ b/packages/ai-openai/src/adapters/image.ts @@ -41,7 +41,17 @@ const EDIT_MAX_IMAGES: Record = { /** * Configuration for OpenAI image adapter */ -export interface OpenAIImageConfig extends OpenAIClientConfig {} +export interface OpenAIImageConfig extends OpenAIClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs for image edits. OpenAI's + * `/images/edits` endpoint requires uploaded file bytes (no URL + * passthrough), so an HTTP(S) URL has to be downloaded and buffered in + * memory — which can OOM constrained runtimes (e.g. Cloudflare Workers). + * When `false` (the default), HTTP(S) URL image inputs throw; pass a `data:` + * URI, or set this to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * OpenAI Image Generation Adapter @@ -67,10 +77,13 @@ export class OpenAIImageAdapter< readonly name = 'openai' as const protected client: OpenAI + private readonly allowUrlFetch: boolean constructor(config: OpenAIImageConfig, model: TModel) { super(model, {}) - this.client = new OpenAI(config) + const { allowUrlFetch, ...clientOptions } = config + this.client = new OpenAI(clientOptions) + this.allowUrlFetch = allowUrlFetch ?? false } async generateImages( @@ -226,11 +239,13 @@ export class OpenAIImageAdapter< } const sourceFiles = await Promise.all( - sourceParts.map((part, i) => imagePartToFile(part, `source-${i}`)), + sourceParts.map((part, i) => + imagePartToFile(part, `source-${i}`, this.allowUrlFetch), + ), ) const [firstSourceFile] = sourceFiles const maskFile = maskParts[0] - ? await imagePartToFile(maskParts[0], 'mask') + ? await imagePartToFile(maskParts[0], 'mask', this.allowUrlFetch) : undefined // `modelOptions` is typed across all four image models (including dall-e-3's diff --git a/packages/ai-openai/src/adapters/video.ts b/packages/ai-openai/src/adapters/video.ts index e28a34be5..3e76478d6 100644 --- a/packages/ai-openai/src/adapters/video.ts +++ b/packages/ai-openai/src/adapters/video.ts @@ -48,7 +48,17 @@ function warnIfLargeMediaBuffer(byteLength: number, source: string): void { * * @experimental Video generation is an experimental feature and may change. */ -export interface OpenAIVideoConfig extends OpenAIClientConfig {} +export interface OpenAIVideoConfig extends OpenAIClientConfig { + /** + * Opt into fetching HTTP(S) image URL inputs for Sora's `input_reference`. + * The endpoint requires uploaded file bytes (no URL passthrough), so an + * HTTP(S) URL has to be downloaded and buffered in memory — which can OOM + * constrained runtimes (e.g. Cloudflare Workers). When `false` (the + * default), HTTP(S) URL image inputs throw; pass a `data:` URI, or set this + * to `true` to opt into buffering. + */ + allowUrlFetch?: boolean +} /** * OpenAI Video Generation Adapter @@ -84,7 +94,8 @@ export class OpenAIVideoAdapter< // We hold our own typed copy on `clientConfig` and pass an empty object up. super({}, model) this.clientConfig = config - this.client = new OpenAI(config) + const { allowUrlFetch: _allowUrlFetch, ...clientOptions } = config + this.client = new OpenAI(clientOptions) } async createVideoJob( @@ -126,6 +137,7 @@ export class OpenAIVideoAdapter< request.input_reference = await imagePartToFile( inputReference, 'input-reference', + this.clientConfig.allowUrlFetch ?? false, ) } // `VideoCreateParams.size` is `size?: VideoSize` (no `| undefined`), so we diff --git a/packages/ai-openai/src/image/image-input-to-file.ts b/packages/ai-openai/src/image/image-input-to-file.ts index 2074496fd..77c3f2a30 100644 --- a/packages/ai-openai/src/image/image-input-to-file.ts +++ b/packages/ai-openai/src/image/image-input-to-file.ts @@ -27,7 +27,12 @@ function ensureFileSupport(): void { * Convert a TanStack `ImagePart` into an OpenAI-compatible `File`. * * - `source.type === 'data'`: decode base64 → Buffer → File. - * - `source.type === 'url'`: fetch the URL (or parse data: URI) → File. + * - `source.type === 'url'` with a `data:` URI: parse in-memory → File. + * - `source.type === 'url'` with an HTTP(S) URL: fetch → File, but only when + * `allowUrlFetch` is set. OpenAI's `/images/edits` and Sora + * `input_reference` require real file bytes (no URL passthrough), so the + * image has to be downloaded and buffered in memory — which can OOM + * constrained runtimes. Off by default; the caller opts in. * * The mime type comes from the source when available, else inferred from the * URL extension, else `image/png`. @@ -35,6 +40,7 @@ function ensureFileSupport(): void { export async function imagePartToFile( part: ImagePart, fallbackName: string, + allowUrlFetch: boolean, ): Promise { ensureFileSupport() @@ -46,6 +52,19 @@ export async function imagePartToFile( }) } + // Remote HTTP(S) URLs must be downloaded and buffered before upload; gate + // that behind an explicit opt-in. `data:` URIs are already in memory, so + // they're handled uniformly via fetch() below without the flag. + if (/^https?:\/\//i.test(part.source.value) && !allowUrlFetch) { + throw new Error( + `openai: HTTP(S) URL image inputs are not fetched by default because ` + + `OpenAI's edit / input_reference endpoints require uploaded bytes, so ` + + `the image would be downloaded and buffered in memory (risking OOM on ` + + `constrained runtimes). Pass a data: URI, or set \`allowUrlFetch: true\` ` + + `on the adapter config to opt into fetching. URL: ${part.source.value}`, + ) + } + // URL source — also handles data: URIs uniformly via fetch(). const response = await fetch(part.source.value) if (!response.ok) { diff --git a/packages/ai-openai/tests/image-adapter.test.ts b/packages/ai-openai/tests/image-adapter.test.ts index ffae82297..010721080 100644 --- a/packages/ai-openai/tests/image-adapter.test.ts +++ b/packages/ai-openai/tests/image-adapter.test.ts @@ -297,6 +297,94 @@ describe('OpenAI Image Adapter', () => { expect(result.images[0]!.b64Json).toBe('edited-base64') }) + it('throws on an HTTP(S) URL image input by default instead of buffering it (#907)', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key' }, + 'gpt-image-1', + ) + const editSpy = adapter.spyOnImagesEdit() + + await expect( + adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(editSpy).not.toHaveBeenCalled() + }) + + it('fetches HTTP(S) URL image inputs when allowUrlFetch is set', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key', allowUrlFetch: true }, + 'gpt-image-1', + ) + const editSpy = adapter + .spyOnImagesEdit() + .mockResolvedValueOnce(imagesEditResponse) + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + const result = await adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' }, + }, + ], + logger: testLogger, + }) + expect(fetchMock).toHaveBeenCalledWith('https://example.com/photo.jpg') + expect(editSpy).toHaveBeenCalledTimes(1) + expect(editSpy.mock.calls[0]![0].image).toBeInstanceOf(File) + expect(result.images[0]!.b64Json).toBe('edited-base64') + } finally { + vi.unstubAllGlobals() + } + }) + + it('accepts a data: URI URL source without allowUrlFetch (in-memory, no network)', async () => { + const adapter = new TestOpenAIImageAdapter( + { apiKey: 'test-api-key' }, + 'gpt-image-1', + ) + const editSpy = adapter + .spyOnImagesEdit() + .mockResolvedValueOnce(imagesEditResponse) + + const result = await adapter.generateImages({ + model: 'gpt-image-1', + prompt: [ + { type: 'text', content: 'Make it cinematic' }, + { + type: 'image', + source: { + type: 'url', + value: 'data:image/png;base64,aGVsbG8=', + }, + }, + ], + logger: testLogger, + }) + + expect(editSpy).toHaveBeenCalledTimes(1) + expect(editSpy.mock.calls[0]![0].image).toBeInstanceOf(File) + expect(result.images[0]!.b64Json).toBe('edited-base64') + }) + it('rejects dall-e-3 with a clear error when the prompt has image parts', async () => { const adapter = new TestOpenAIImageAdapter( { apiKey: 'test-api-key' }, diff --git a/packages/ai-openai/tests/video-adapter.test.ts b/packages/ai-openai/tests/video-adapter.test.ts index 5d28241a8..158b85ce8 100644 --- a/packages/ai-openai/tests/video-adapter.test.ts +++ b/packages/ai-openai/tests/video-adapter.test.ts @@ -51,6 +51,60 @@ describe('OpenAI Video Adapter', () => { expect(result.model).toBe('sora-2') }) + it('throws on an HTTP(S) URL input_reference by default instead of buffering it (#907)', async () => { + const { adapter, mockCreate } = mockedAdapter() + + await expect( + adapter.createVideoJob({ + model: 'sora-2', + prompt: [ + { type: 'text', content: 'Slow cinematic push-in' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/ref.jpg' }, + }, + ], + logger: testLogger, + }), + ).rejects.toThrow(/allowUrlFetch/) + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('fetches an HTTP(S) URL input_reference when allowUrlFetch is set', async () => { + const adapter = createOpenaiVideo('sora-2', 'test-api-key', { + allowUrlFetch: true, + }) + const mockCreate = vi.fn().mockResolvedValue({ id: 'video-job-2' }) + ;(adapter as unknown as { client: { videos: unknown } }).client = { + videos: { create: mockCreate }, + } + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([104, 105]), { + headers: { 'content-type': 'image/jpeg' }, + }), + ) + vi.stubGlobal('fetch', fetchMock) + + try { + await adapter.createVideoJob({ + model: 'sora-2', + prompt: [ + { type: 'text', content: 'Slow cinematic push-in' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/ref.jpg' }, + }, + ], + logger: testLogger, + }) + } finally { + vi.unstubAllGlobals() + } + + expect(fetchMock).toHaveBeenCalledWith('https://example.com/ref.jpg') + expect(mockCreate.mock.calls[0]![0].input_reference).toBeInstanceOf(File) + }) + it('throws when more than one image part is provided', async () => { const { adapter, mockCreate } = mockedAdapter() diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index b55638111..3c5258981 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -261,6 +261,17 @@ await generateVideo({ }) ``` +**URL inputs that require an upload throw by default.** Most adapters pass a +`type: 'url'` source straight through to the provider. Three paths can't — +OpenAI `images.edit()`, OpenAI Sora `input_reference`, and Gemini **Veo** — +because the provider only accepts uploaded bytes (Veo also takes a `gs://` +reference). For those, an HTTP(S) URL would have to be downloaded and buffered +in memory, which can OOM constrained runtimes, so they **throw** on an HTTP(S) +URL image input by default. Pass a `data:` URI (or `gs://` for Veo), or opt in +with `allowUrlFetch: true` on the adapter config +(`createOpenaiImage(model, apiKey, { allowUrlFetch: true })`, and likewise on +`createOpenaiVideo` / `createGeminiVideo`). `data:` URIs never need the flag. + **Role hints** (`metadata.role`): | Role | Maps to |