feat (provider/luma): add Luma provider (#4516)

Author: shaper

.changeset/thin-rice-drum.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/luma': patch
+---
+
+feat (provider/luma): add Luma provider

.changeset/weak-bobcats-wink.md

@@ -0,0 +1,6 @@
+---
+'@ai-sdk/provider-utils': patch
+'@ai-sdk/fireworks': patch
+---
+
+feat (provider-utils): add getFromApi and response handlers for binary responses and status-code errors

CHANGELOG.md

@@ -17,6 +17,7 @@ You can find the changelogs for the individual packages in their respective `CHA
 - [@ai-sdk/google](./packages/google/CHANGELOG.md)
 - [@ai-sdk/google-vertex](./packages/google-vertex/CHANGELOG.md)
 - [@ai-sdk/groq](./packages/groq/CHANGELOG.md)
+- [@ai-sdk/luma](./packages/luma/CHANGELOG.md)
 - [@ai-sdk/mistral](./packages/mistral/CHANGELOG.md)
 - [@ai-sdk/openai](./packages/openai/CHANGELOG.md)
 - [@ai-sdk/openai-compatible](./packages/openai-compatible/CHANGELOG.md)

content/docs/03-ai-sdk-core/35-image-generation.mdx

@@ -225,3 +225,5 @@ try {
 | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models)         | `accounts/fireworks/models/playground-v2-1024px-aesthetic`   | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640                                                                           |
 | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models)         | `accounts/fireworks/models/SSD-1B`                           | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640                                                                           |
 | [Fireworks](/providers/ai-sdk-providers/fireworks#image-models)         | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0`    | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640                                                                           |
+| [Luma](/providers/ai-sdk-providers/luma#image-models)                   | `photon-1`                                                   | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9                                                                                                                               |
+| [Luma](/providers/ai-sdk-providers/luma#image-models)                   | `photon-flash-1`                                             | 1:1, 3:4, 4:3, 9:16, 16:9, 9:21, 21:9                                                                                                                               |

content/providers/01-ai-sdk-providers/80-luma.mdx

@@ -0,0 +1,243 @@
+---
+title: Luma
+description: Learn how to use Luma AI models with the AI SDK.
+---
+
+# Luma Provider
+
+[Luma AI](https://lumalabs.ai/) provides state-of-the-art image generation models through their Dream Machine platform. Their models offer ultra-high quality image generation with superior prompt understanding and unique capabilities like character consistency and multi-image reference support.
+
+## Setup
+
+The Luma provider is available via the `@ai-sdk/luma` module. You can install it with
+
+<Tabs items={['pnpm', 'npm', 'yarn']}>
+  <Tab>
+    <Snippet text="pnpm add @ai-sdk/luma" dark />
+  </Tab>
+  <Tab>
+    <Snippet text="npm install @ai-sdk/luma" dark />
+  </Tab>
+  <Tab>
+    <Snippet text="yarn add @ai-sdk/luma" dark />
+  </Tab>
+</Tabs>
+
+## Provider Instance
+
+You can import the default provider instance `luma` from `@ai-sdk/luma`:
+
+```ts
+import { luma } from '@ai-sdk/luma';
+```
+
+If you need a customized setup, you can import `createLuma` and create a provider instance with your settings:
+
+```ts
+import { createLuma } from '@ai-sdk/luma';
+
+const luma = createLuma({
+  apiKey: 'your-api-key', // optional, defaults to LUMA_API_KEY environment variable
+  baseURL: 'custom-url', // optional
+  headers: {
+    /* custom headers */
+  }, // optional
+});
+```
+
+You can use the following optional settings to customize the Luma provider instance:
+
+- **baseURL** _string_
+
+  Use a different URL prefix for API calls, e.g. to use proxy servers.
+  The default prefix is `https://api.lumalabs.ai`.
+
+- **apiKey** _string_
+
+  API key that is being sent using the `Authorization` header.
+  It defaults to the `LUMA_API_KEY` environment variable.
+
+- **headers** _Record&lt;string,string&gt;_
+
+  Custom headers to include in the requests.
+
+- **fetch** _(input: RequestInfo, init?: RequestInit) => Promise&lt;Response&gt;_
+
+  Custom [fetch](https://developer.mozilla.org/en-US/docs/Web/API/fetch) implementation.
+  You can use it as a middleware to intercept requests,
+  or to provide a custom fetch implementation for e.g. testing.
+
+## Image Models
+
+You can create Luma image models using the `.image()` factory method.
+For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image).
+
+### Basic Usage
+
+```ts
+import { luma } from '@ai-sdk/luma';
+import { experimental_generateImage as generateImage } from 'ai';
+import fs from 'fs';
+
+const { image } = await generateImage({
+  model: luma.image('photon-1'),
+  prompt: 'A serene mountain landscape at sunset',
+  aspectRatio: '16:9',
+});
+
+const filename = `image-${Date.now()}.png`;
+fs.writeFileSync(filename, image.uint8Array);
+console.log(`Image saved to ${filename}`);
+```
+
+### Image Model Settings
+
+When creating an image model, you can customize the generation behavior with optional settings:
+
+```ts
+const model = luma.image('photon-1', {
+  maxImagesPerCall: 1, // Maximum number of images to generate per API call
+  pollIntervalMillis: 5000, // How often to check for completed images (in ms)
+  maxPollAttempts: 10, // Maximum number of polling attempts before timeout
+});
+```
+
+Since Luma processes images through an asynchronous queue system, these settings allow you to tune the polling behavior:
+
+- **maxImagesPerCall** _number_
+
+  Override the maximum number of images generated per API call. Defaults to 1.
+
+- **pollIntervalMillis** _number_
+
+  Control how frequently the API is checked for completed images while they are
+  being processed. Defaults to 500ms.
+
+- **maxPollAttempts** _number_
+
+  Limit how long to wait for results before timing out, since image generation
+  is queued asynchronously. Defaults to 120 attempts.
+
+### Model Capabilities
+
+Luma offers two main models:
+
+| Model            | Description                                                      |
+| ---------------- | ---------------------------------------------------------------- |
+| `photon-1`       | High-quality image generation with superior prompt understanding |
+| `photon-flash-1` | Faster generation optimized for speed while maintaining quality  |
+
+Both models support the following aspect ratios:
+
+- 1:1
+- 3:4
+- 4:3
+- 9:16
+- 16:9 (default)
+- 9:21
+- 21:9
+
+For more details about supported aspect ratios, see the [Luma Image Generation documentation](https://docs.lumalabs.ai/docs/image-generation).
+
+Key features of Luma models include:
+
+- Ultra-high quality image generation
+- 10x higher cost efficiency compared to similar models
+- Superior prompt understanding and adherence
+- Unique character consistency capabilities from single reference images
+- Multi-image reference support for precise style matching
+
+### Advanced Options
+
+Luma models support several advanced features through the `providerOptions.luma` parameter.
+
+#### Image Reference
+
+Use up to 4 reference images to guide your generation. Useful for creating variations or visualizing complex concepts. Adjust the `weight` (0-1) to control the influence of reference images.
+
+```ts
+// Example: Generate a salamander with reference
+await generateImage({
+  model: luma.image('photon-1'),
+  prompt: 'A salamander at dusk in a forest pond, in the style of ukiyo-e',
+  providerOptions: {
+    luma: {
+      image_ref: [
+        {
+          url: 'https://example.com/reference.jpg',
+          weight: 0.85,
+        },
+      ],
+    },
+  },
+});
+```
+
+#### Style Reference
+
+Apply specific visual styles to your generations using reference images. Control the style influence using the `weight` parameter.
+
+```ts
+// Example: Generate with style reference
+await generateImage({
+  model: luma.image('photon-1'),
+  prompt: 'A blue cream Persian cat launching its website on Vercel',
+  providerOptions: {
+    luma: {
+      style_ref: [
+        {
+          url: 'https://example.com/style.jpg',
+          weight: 0.8,
+        },
+      ],
+    },
+  },
+});
+```
+
+#### Character Reference
+
+Create consistent and personalized characters using up to 4 reference images of the same subject. More reference images improve character representation.
+
+```ts
+// Example: Generate character-based image
+await generateImage({
+  model: luma.image('photon-1'),
+  prompt: 'A woman with a cat riding a broomstick in a forest',
+  providerOptions: {
+    luma: {
+      character_ref: {
+        identity0: {
+          images: ['https://example.com/character.jpg'],
+        },
+      },
+    },
+  },
+});
+```
+
+#### Modify Image
+
+Transform existing images using text prompts. Use the `weight` parameter to control how closely the result matches the input image (higher weight = closer to input but less creative).
+
+<Note>
+  For color changes, it's recommended to use a lower weight value (0.0-0.1).
+</Note>
+
+```ts
+// Example: Modify existing image
+await generateImage({
+  model: luma.image('photon-1'),
+  prompt: 'transform the bike to a boat',
+  providerOptions: {
+    luma: {
+      modify_image_ref: {
+        url: 'https://example.com/image.jpg',
+        weight: 1.0,
+      },
+    },
+  },
+});
+```
+
+For more details about Luma's capabilities and features, visit the [Luma Image Generation documentation](https://docs.lumalabs.ai/docs/image-generation).

examples/ai-core/package.json

@@ -14,6 +14,7 @@
     "@ai-sdk/google": "1.1.2",
     "@ai-sdk/google-vertex": "2.1.2",
     "@ai-sdk/groq": "1.1.2",
+    "@ai-sdk/luma": "0.0.0",
     "@ai-sdk/mistral": "1.1.2",
     "@ai-sdk/openai": "1.1.2",
     "@ai-sdk/openai-compatible": "0.1.3",

examples/ai-core/src/e2e/feature-test-suite.ts

@@ -77,6 +77,7 @@ export interface ModelVariants {
   invalidModel?: LanguageModelV1;
   languageModels?: ModelWithCapabilities<LanguageModelV1>[];
   embeddingModels?: ModelWithCapabilities<EmbeddingModelV1<string>>[];
+  invalidImageModel?: ImageModelV1;
   imageModels?: ModelWithCapabilities<ImageModelV1>[];
 }
 
@@ -1021,6 +1022,24 @@ export function createFeatureTestSuite({
         });
       }
 
+      if (models.invalidImageModel) {
+        describe('Image Model Error Handling:', () => {
+          const invalidModel = models.invalidImageModel!;
+
+          it('should throw error on generate image attempt with invalid model ID', async () => {
+            try {
+              await generateImage({
+                model: invalidModel,
+                prompt: 'This should fail',
+              });
+            } catch (error) {
+              expect(error).toBeInstanceOf(APICallError);
+              errorValidator(error as APICallError);
+            }
+          });
+        });
+      }
+
       if (models.embeddingModels && models.embeddingModels.length > 0) {
         describe.each(createModelObjects(models.embeddingModels))(
           'Embedding Model: $modelId',

examples/ai-core/src/e2e/luma.test.ts

@@ -0,0 +1,27 @@
+import { expect } from 'vitest';
+import { luma as provider, LumaErrorData } from '@ai-sdk/luma';
+import { APICallError } from '@ai-sdk/provider';
+import {
+  createFeatureTestSuite,
+  createImageModelWithCapabilities,
+} from './feature-test-suite';
+import 'dotenv/config';
+
+createFeatureTestSuite({
+  name: 'Luma',
+  models: {
+    invalidImageModel: provider.image('no-such-model'),
+    imageModels: [
+      createImageModelWithCapabilities(provider.image('photon-flash-1')),
+      createImageModelWithCapabilities(provider.image('photon-1')),
+    ],
+  },
+  timeout: 30000,
+  customAssertions: {
+    errorValidator: (error: APICallError) => {
+      expect((error.data as LumaErrorData).detail[0].msg).toMatch(
+        /Input should be/i,
+      );
+    },
+  },
+})();

examples/ai-core/src/generate-image/luma-character-reference.ts

@@ -0,0 +1,32 @@
+import { luma } from '@ai-sdk/luma';
+import { experimental_generateImage as generateImage } from 'ai';
+import 'dotenv/config';
+import fs from 'fs';
+
+async function main() {
+  const result = await generateImage({
+    model: luma.image('photon-flash-1'),
+    prompt: 'A woman with a cat riding a broomstick in a forest',
+    aspectRatio: '1:1',
+    providerOptions: {
+      luma: {
+        // https://docs.lumalabs.ai/docs/image-generation#character-reference
+        character_ref: {
+          identity0: {
+            images: [
+              'https://hebbkx1anhila5yf.public.blob.vercel-storage.com/future-me-8hcBWcZOkbE53q3gshhEm16S87qDpF.jpeg',
+            ],
+          },
+        },
+      },
+    },
+  });
+
+  for (const [index, image] of result.images.entries()) {
+    const filename = `image-${Date.now()}-${index}.png`;
+    fs.writeFileSync(filename, image.uint8Array);
+    console.log(`Image saved to ${filename}`);
+  }
+}
+
+main().catch(console.error);