Skip to content

Commit 6636db6

Browse files
lgrammelzeke
lgrammel
and
zeke
authoredJan 9, 2025··
feat (provider/replicate): add replicate image provider (#4311)
Co-authored-by: Zeke Sikelianos <[email protected]>
·
@ai-sdk/xai@1.2.5ai@4.0.31
1 parent e6dfef4 commit 6636db6

29 files changed

+1022
-55
lines changed
 

‎.changeset/giant-tigers-jump.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/replicate': patch
3+
---
4+
5+
feat (provider/replicate): add replicate image provider

‎.changeset/tricky-wombats-attack.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@ai-sdk/provider-utils': patch
3+
---
4+
5+
feat (provider-utils): add unified test server

‎content/docs/03-ai-sdk-core/35-image-generation.mdx

+15-15
Original file line numberDiff line numberDiff line change
@@ -166,18 +166,18 @@ const { image, warnings } = await generateImage({
166166

167167
## Image Models
168168

169-
| Provider | Model | Sizes | Aspect Ratios |
170-
| ----------------------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------- | ----------------------------------------------- |
171-
| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 |
172-
| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 |
173-
| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | use size |
174-
| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | use size |
175-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-dev-fp8` | Use aspect ratio | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |
176-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-schnell-fp8` | Use aspect ratio | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |
177-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-5-1024px-aesthetic` | 640x1536 to 1536x640\* | use size |
178-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/japanese-stable-diffusion-xl` | 640x1536 to 1536x640\* | use size |
179-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-1024px-aesthetic` | 640x1536 to 1536x640\* | use size |
180-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/SSD-1B` | 640x1536 to 1536x640\* | use size |
181-
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0` | 640x1536 to 1536x640\* | use size |
182-
183-
\* Supported sizes: 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640
169+
| Provider | Model | Support sizes (`width x height`) or aspect ratios (`width : height`) |
170+
| ----------------------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
171+
| [Replicate](/providers/ai-sdk-providers/replicate) | `black-forest-labs/flux-schnell` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |
172+
| [Replicate](/providers/ai-sdk-providers/replicate) | `recraft-ai/recraft-v3` | 1024x1024, 1365x1024, 1024x1365, 1536x1024, 1024x1536, 1820x1024, 1024x1820, 1024x2048, 2048x1024, 1434x1024, 1024x1434, 1024x1280, 1280x1024, 1024x1707, 1707x1024 |
173+
| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 |
174+
| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 |
175+
| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 |
176+
| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 |
177+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-dev-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |
178+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-schnell-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |
179+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-5-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 |
180+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/japanese-stable-diffusion-xl` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 |
181+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/playground-v2-1024px-aesthetic` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 |
182+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/SSD-1B` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 |
183+
| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/stable-diffusion-xl-1024-v1-0` | 640x1536, 768x1344, 832x1216, 896x1152, 1024x1024, 1152x896, 1216x832, 1344x768, 1536x640 |

‎content/providers/01-ai-sdk-providers/60-replicate.mdx

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
---
2+
title: Replicate
3+
description: Learn how to use Replicate models with the AI SDK.
4+
---
5+
6+
# Replicate Provider
7+
8+
[Replicate](https://replicate.com/) is a platform for running open-source AI models.
9+
It is a popular choice for running image generation models.
10+
11+
## Setup
12+
13+
The Replicate provider is available via the `@ai-sdk/replicate` module. You can install it with
14+
15+
<Tabs items={['pnpm', 'npm', 'yarn']}>
16+
<Tab>
17+
<Snippet text="pnpm add @ai-sdk/replicate" dark />
18+
</Tab>
19+
<Tab>
20+
<Snippet text="npm install @ai-sdk/replicate" dark />
21+
</Tab>
22+
<Tab>
23+
<Snippet text="yarn add @ai-sdk/replicate" dark />
24+
</Tab>
25+
</Tabs>
26+
27+
## Provider Instance
28+
29+
You can import the default provider instance `replicate` from `@ai-sdk/replicate`:
30+
31+
```ts
32+
import { replicate } from '@ai-sdk/replicate';
33+
```
34+
35+
If you need a customized setup, you can import `createReplicate` from `@ai-sdk/replicate`
36+
and create a provider instance with your settings:
37+
38+
```ts
39+
import { createReplicate } from '@ai-sdk/replicate';
40+
41+
const replicate = createReplicate({
42+
apiKey: process.env.REPLICATE_API_KEY ?? '',
43+
});
44+
```
45+
46+
You can use the following optional settings to customize the Replicate provider instance:
47+
48+
- **baseURL** _string_
49+
50+
Use a different URL prefix for API calls, e.g. to use proxy servers.
51+
The default prefix is `https://api.replicate.com/v1`.
52+
53+
- **apiToken** _string_
54+
55+
API token that is being sent using the `Authorization` header. It defaults to
56+
the `REPLICATE_API_TOKEN` environment variable.
57+
58+
- **headers** _Record&lt;string,string&gt;_
59+
60+
Custom headers to include in the requests.
61+
62+
- **fetch** _(input: RequestInfo, init?: RequestInit) => Promise&lt;Response&gt;_
63+
64+
Custom [fetch](https://developer.mozilla.org/en-US/docs/Web/API/fetch) implementation.
65+
66+
## Image Models
67+
68+
You can create Replicate image models using the `.image()` factory method.
69+
For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image).
70+
71+
<Note>
72+
Model support for `size` and other parameters varies by model. Check the
73+
model's documentation on [Replicate](https://replicate.com/explore) for
74+
supported options and additional parameters that can be passed via
75+
`providerOptions.replicate`.
76+
</Note>
77+
78+
### Basic Usage
79+
80+
```ts
81+
import { replicate } from '@ai-sdk/replicate';
82+
import { experimental_generateImage as generateImage } from 'ai';
83+
84+
const { image } = await generateImage({
85+
model: replicate.image('black-forest-labs/flux-schnell'),
86+
prompt: 'The Loch Ness Monster getting a manicure',
87+
aspectRatio: '16:9',
88+
});
89+
```
90+
91+
### Model-specific options
92+
93+
```ts highlight="9-11"
94+
import { replicate } from '@ai-sdk/replicate';
95+
import { experimental_generateImage as generateImage } from 'ai';
96+
97+
const { image } = await generateImage({
98+
model: replicate.image('recraft-ai/recraft-v3'),
99+
prompt: 'The Loch Ness Monster getting a manicure',
100+
size: '1365x1024',
101+
providerOptions: {
102+
replicate: {
103+
style: 'realistic_image',
104+
},
105+
},
106+
});
107+
```
108+
109+
For more details, see the [Replicate models page](https://replicate.com/explore).

‎examples/ai-core/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
"@ai-sdk/openai": "1.0.15",
1818
"@ai-sdk/openai-compatible": "0.0.14",
1919
"@ai-sdk/provider": "1.0.4",
20+
"@ai-sdk/replicate": "0.0.0",
2021
"@ai-sdk/togetherai": "0.0.17",
2122
"@ai-sdk/xai": "1.0.15",
2223
"@opentelemetry/sdk-node": "0.54.2",

‎examples/ai-core/src/generate-image/replicate-1.ts

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { replicate } from '@ai-sdk/replicate';
2+
import { experimental_generateImage as generateImage } from 'ai';
3+
import 'dotenv/config';
4+
import fs from 'node:fs';
5+
6+
async function main() {
7+
const { image } = await generateImage({
8+
model: replicate.image('black-forest-labs/flux-schnell'),
9+
prompt: 'The Loch Ness Monster getting a manicure',
10+
});
11+
12+
const filename = `image-${Date.now()}.png`;
13+
fs.writeFileSync(filename, image.uint8Array);
14+
console.log(`Image saved to ${filename}`);
15+
}
16+
17+
main().catch(console.error);

‎examples/ai-core/src/generate-image/replicate-2.ts

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { replicate } from '@ai-sdk/replicate';
2+
import { experimental_generateImage as generateImage } from 'ai';
3+
import 'dotenv/config';
4+
import fs from 'node:fs';
5+
6+
async function main() {
7+
const { image } = await generateImage({
8+
model: replicate.image('black-forest-labs/flux-schnell'),
9+
prompt: 'The Loch Ness Monster getting a manicure',
10+
aspectRatio: '16:9',
11+
});
12+
13+
const filename = `image-${Date.now()}.png`;
14+
fs.writeFileSync(filename, image.uint8Array);
15+
console.log(`Image saved to ${filename}`);
16+
}
17+
18+
main().catch(console.error);

‎examples/ai-core/src/generate-image/replicate-3.ts

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { replicate } from '@ai-sdk/replicate';
2+
import { experimental_generateImage as generateImage } from 'ai';
3+
import 'dotenv/config';
4+
import fs from 'node:fs';
5+
6+
async function main() {
7+
const { image } = await generateImage({
8+
model: replicate.image('recraft-ai/recraft-v3'),
9+
prompt: 'The Loch Ness Monster getting a manicure',
10+
size: '1365x1024',
11+
providerOptions: {
12+
replicate: {
13+
style: 'realistic_image',
14+
},
15+
},
16+
});
17+
18+
const filename = `image-${Date.now()}.png`;
19+
fs.writeFileSync(filename, image.uint8Array);
20+
console.log(`Image saved to ${filename}`);
21+
}
22+
23+
main().catch(console.error);

‎packages/provider-utils/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"devDependencies": {
4646
"@types/node": "^18",
4747
"@vercel/ai-tsconfig": "workspace:*",
48-
"msw": "2.6.4",
48+
"msw": "2.7.0",
4949
"tsup": "^8",
5050
"typescript": "5.6.3",
5151
"zod": "3.23.8"

‎packages/provider-utils/src/test/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ export * from './convert-response-stream-to-array';
77
export * from './json-test-server';
88
export * from './streaming-test-server';
99
export * from './test-server';
10+
export * from './unified-test-server';

‎packages/provider-utils/src/test/test-server.ts

+2
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ function createServer({
9999
});
100100

101101
// keep track of url invocation counts:
102+
// TODO bug needs reset after each test
102103
const urlInvocationCounts = Object.fromEntries(
103104
Object.entries(responsesByUrl).map(([url]) => [url, 0]),
104105
);
@@ -111,6 +112,7 @@ function createServer({
111112
const invocationCount = urlInvocationCounts[url]++;
112113
const response =
113114
responses[
115+
// TODO bug needs to be >=
114116
invocationCount > responses.length
115117
? responses.length - 1
116118
: invocationCount

0 commit comments

Comments
 (0)
Please sign in to comment.