feat (provider): add metadata extraction mechanism to openai-compatible providers (#4397)

Co-authored-by: kongmoumou <[email protected]>
Co-authored-by: Lars Grammel <[email protected]>

Author: 3 people

.changeset/big-bulldogs-behave.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/deepseek': patch
+---
+
+feat (provider/deepseek): extract cache usage as provide metadata

.changeset/curvy-ears-exercise.md

@@ -0,0 +1,6 @@
+---
+'@ai-sdk/openai-compatible': patch
+'@ai-sdk/provider-utils': patch
+---
+
+feat (provider): add metadata extraction mechanism to openai-compatible providers

content/providers/02-openai-compatible-providers/index.mdx

@@ -170,3 +170,96 @@ const provider = createOpenAICompatible({
 
 For example, with the above configuration, API requests would include the query parameter in the URL like:
 `https://api.provider.com/v1/chat/completions?api-version=1.0.0`.
+
+## Custom Metadata Extraction
+
+The OpenAI Compatible provider supports extracting provider-specific metadata from API responses through metadata extractors.
+These extractors allow you to capture additional information returned by the provider beyond the standard response format.
+
+Metadata extractors receive the raw, unprocessed response data from the provider, giving you complete flexibility
+to extract any custom fields or experimental features that the provider may include.
+This is particularly useful when:
+
+- Working with providers that include non-standard response fields
+- Experimenting with beta or preview features
+- Capturing provider-specific metrics or debugging information
+- Supporting rapid provider API evolution without SDK changes
+
+Metadata extractors work with both streaming and non-streaming chat completions and consist of two main components:
+
+1. A function to extract metadata from complete responses
+2. A streaming extractor that can accumulate metadata across chunks in a streaming response
+
+Here's an example metadata extractor that captures both standard and custom provider data:
+
+```typescript
+const MyMetadataExtractor: MetadataExtractor = {
+  // Process complete, non-streaming responses
+  extractMetadata: ({ parsedBody }) => {
+    // You have access to the complete raw response
+    // Extract any fields the provider includes
+    return {
+      myProvider: {
+        standardUsage: parsedBody.usage,
+        experimentalFeatures: parsedBody.beta_features,
+        customMetrics: {
+          processingTime: parsedBody.server_timing?.total_ms,
+          modelVersion: parsedBody.model_version,
+          // ... any other provider-specific data
+        },
+      },
+    };
+  },
+
+  // Process streaming responses
+  createStreamExtractor: () => {
+    let accumulatedData = {
+      timing: [],
+      customFields: {},
+    };
+
+    return {
+      // Process each chunk's raw data
+      processChunk: parsedChunk => {
+        if (parsedChunk.server_timing) {
+          accumulatedData.timing.push(parsedChunk.server_timing);
+        }
+        if (parsedChunk.custom_data) {
+          Object.assign(accumulatedData.customFields, parsedChunk.custom_data);
+        }
+      },
+      // Build final metadata from accumulated data
+      buildMetadata: () => ({
+        myProvider: {
+          streamTiming: accumulatedData.timing,
+          customData: accumulatedData.customFields,
+        },
+      }),
+    };
+  },
+};
+```
+
+You can provide a metadata extractor when creating your provider instance:
+
+```typescript
+const provider = createOpenAICompatible({
+  name: 'my-provider',
+  apiKey: process.env.PROVIDER_API_KEY,
+  baseURL: 'https://api.provider.com/v1',
+  metadataExtractor: MyMetadataExtractor,
+});
+```
+
+The extracted metadata will be included in the response under the `providerMetadata` field:
+
+```typescript
+const { text, providerMetadata } = await generateText({
+  model: provider('model-id'),
+  prompt: 'Hello',
+});
+
+console.log(providerMetadata.myProvider.customMetric);
+```
+
+This allows you to access provider-specific information while maintaining a consistent interface across different providers.

examples/ai-core/src/generate-text/deepseek-cache-token.ts

@@ -0,0 +1,38 @@
+import { deepseek } from '@ai-sdk/deepseek';
+import { generateText } from 'ai';
+import 'dotenv/config';
+import fs from 'node:fs';
+
+const errorMessage = fs.readFileSync('data/error-message.txt', 'utf8');
+
+async function main() {
+  const result = await generateText({
+    model: deepseek.chat('deepseek-chat'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'You are a JavaScript expert.',
+          },
+          {
+            type: 'text',
+            text: `Error message: ${errorMessage}`,
+          },
+          {
+            type: 'text',
+            text: 'Explain the error message.',
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.text);
+  console.log(result.usage);
+  console.log(result.experimental_providerMetadata);
+  // "prompt_cache_hit_tokens":1856,"prompt_cache_miss_tokens":5}
+}
+
+main().catch(console.error);

examples/ai-core/src/stream-text/deepseek-cache-token.ts

@@ -0,0 +1,43 @@
+import { deepseek } from '@ai-sdk/deepseek';
+import { streamText } from 'ai';
+import 'dotenv/config';
+import fs from 'node:fs';
+
+const errorMessage = fs.readFileSync('data/error-message.txt', 'utf8');
+
+async function main() {
+  const result = streamText({
+    model: deepseek('deepseek-chat'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'You are a JavaScript expert.',
+          },
+          {
+            type: 'text',
+            text: `Error message: ${errorMessage}`,
+          },
+          {
+            type: 'text',
+            text: 'Explain the error message.',
+          },
+        ],
+      },
+    ],
+  });
+
+  for await (const textPart of result.textStream) {
+    process.stdout.write(textPart);
+  }
+
+  console.log();
+  console.log('Token usage:', await result.usage);
+  console.log('Finish reason:', await result.finishReason);
+  console.log('Provider metadata:', await result.experimental_providerMetadata);
+  // "prompt_cache_hit_tokens":1856,"prompt_cache_miss_tokens":5}
+}
+
+main().catch(console.error);

packages/deepseek/src/deepseek-metadata-extractor.test.ts

@@ -0,0 +1,149 @@
+import { deepSeekMetadataExtractor } from './deepseek-metadata-extractor';
+
+describe('buildMetadataFromResponse', () => {
+  it('should extract metadata from complete response with usage data', () => {
+    const response = {
+      usage: {
+        prompt_cache_hit_tokens: 100,
+        prompt_cache_miss_tokens: 50,
+      },
+    };
+
+    const metadata = deepSeekMetadataExtractor.extractMetadata({
+      parsedBody: response,
+    });
+
+    expect(metadata).toEqual({
+      deepseek: {
+        promptCacheHitTokens: 100,
+        promptCacheMissTokens: 50,
+      },
+    });
+  });
+
+  it('should handle missing usage data', () => {
+    const response = {
+      id: 'test-id',
+      choices: [],
+    };
+
+    const metadata = deepSeekMetadataExtractor.extractMetadata({
+      parsedBody: response,
+    });
+
+    expect(metadata).toBeUndefined();
+  });
+
+  it('should handle invalid response data', () => {
+    const response = 'invalid data';
+
+    const metadata = deepSeekMetadataExtractor.extractMetadata({
+      parsedBody: response,
+    });
+
+    expect(metadata).toBeUndefined();
+  });
+});
+
+describe('streaming metadata processor', () => {
+  it('should process streaming chunks and build final metadata', () => {
+    const processor = deepSeekMetadataExtractor.createStreamExtractor();
+
+    // Process initial chunks without usage data
+    processor.processChunk({
+      choices: [{ finish_reason: null }],
+    });
+
+    // Process final chunk with usage data
+    processor.processChunk({
+      choices: [{ finish_reason: 'stop' }],
+      usage: {
+        prompt_cache_hit_tokens: 100,
+        prompt_cache_miss_tokens: 50,
+      },
+    });
+
+    const finalMetadata = processor.buildMetadata();
+
+    expect(finalMetadata).toEqual({
+      deepseek: {
+        promptCacheHitTokens: 100,
+        promptCacheMissTokens: 50,
+      },
+    });
+  });
+
+  it('should handle streaming chunks without usage data', () => {
+    const processor = deepSeekMetadataExtractor.createStreamExtractor();
+
+    processor.processChunk({
+      choices: [{ finish_reason: 'stop' }],
+    });
+
+    const finalMetadata = processor.buildMetadata();
+
+    expect(finalMetadata).toBeUndefined();
+  });
+
+  it('should handle invalid streaming chunks', () => {
+    const processor = deepSeekMetadataExtractor.createStreamExtractor();
+
+    processor.processChunk('invalid chunk');
+
+    const finalMetadata = processor.buildMetadata();
+
+    expect(finalMetadata).toBeUndefined();
+  });
+
+  it('should only capture usage data from final chunk with stop reason', () => {
+    const processor = deepSeekMetadataExtractor.createStreamExtractor();
+
+    // Process chunk with usage but no stop reason
+    processor.processChunk({
+      choices: [{ finish_reason: null }],
+      usage: {
+        prompt_cache_hit_tokens: 50,
+        prompt_cache_miss_tokens: 25,
+      },
+    });
+
+    // Process final chunk with different usage data
+    processor.processChunk({
+      choices: [{ finish_reason: 'stop' }],
+      usage: {
+        prompt_cache_hit_tokens: 100,
+        prompt_cache_miss_tokens: 50,
+      },
+    });
+
+    const finalMetadata = processor.buildMetadata();
+
+    expect(finalMetadata).toEqual({
+      deepseek: {
+        promptCacheHitTokens: 100,
+        promptCacheMissTokens: 50,
+      },
+    });
+  });
+
+  it('should handle null values in usage data', () => {
+    const processor = deepSeekMetadataExtractor.createStreamExtractor();
+
+    processor.processChunk({
+      choices: [{ finish_reason: 'stop' }],
+      usage: {
+        prompt_cache_hit_tokens: null,
+        prompt_cache_miss_tokens: 50,
+      },
+    });
+
+    const finalMetadata = processor.buildMetadata();
+
+    expect(finalMetadata).toEqual({
+      deepseek: {
+        promptCacheHitTokens: NaN,
+        promptCacheMissTokens: 50,
+      },
+    });
+  });
+});