-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Add support for Base64 encoded images in MistralAI (#10180)
### Motivation and Context Closes #10166 ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
- Loading branch information
1 parent
7a61c9c
commit 704fbdf
Showing
6 changed files
with
92 additions
and
5 deletions.
There are no files selected for viewing
57 changes: 57 additions & 0 deletions
57
dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatCompletion.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using Microsoft.SemanticKernel; | ||
using Microsoft.SemanticKernel.ChatCompletion; | ||
using Microsoft.SemanticKernel.Connectors.MistralAI; | ||
|
||
namespace ChatCompletion; | ||
|
||
// The following example shows how to use Semantic Kernel with MistralAI API | ||
public class MistralAI_ChatCompletion(ITestOutputHelper output) : BaseTest(output) | ||
{ | ||
[Fact] | ||
public async Task GetChatMessageContentAsync() | ||
{ | ||
Assert.NotNull(TestConfiguration.MistralAI.ChatModelId); | ||
Assert.NotNull(TestConfiguration.MistralAI.ApiKey); | ||
|
||
MistralAIChatCompletionService chatService = new( | ||
modelId: TestConfiguration.MistralAI.ChatModelId, | ||
apiKey: TestConfiguration.MistralAI.ApiKey); | ||
|
||
var chatHistory = new ChatHistory("You are a librarian, expert about books"); | ||
|
||
chatHistory.AddUserMessage("Hi, I'm looking for book suggestions"); | ||
this.OutputLastMessage(chatHistory); | ||
|
||
var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 }); | ||
Console.WriteLine(reply); | ||
} | ||
|
||
[Fact] | ||
public async Task GetChatMessageContentUsingImageContentAsync() | ||
{ | ||
Assert.NotNull(TestConfiguration.MistralAI.ImageModelId); | ||
Assert.NotNull(TestConfiguration.MistralAI.ApiKey); | ||
|
||
// Create a logging handler to output HTTP requests and responses | ||
var handler = new LoggingHandler(new HttpClientHandler(), this.Output); | ||
var httpClient = new HttpClient(handler); | ||
|
||
MistralAIChatCompletionService chatService = new( | ||
modelId: TestConfiguration.MistralAI.ImageModelId, | ||
apiKey: TestConfiguration.MistralAI.ApiKey, | ||
httpClient: httpClient); | ||
|
||
var chatHistory = new ChatHistory(); | ||
|
||
var chatMessage = new ChatMessageContent(AuthorRole.User, "What's in this image?"); | ||
chatMessage.Items.Add(new ImageContent("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEA2ADYAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAQABADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5rooor8DP9oD/2Q==")); | ||
|
||
chatHistory.Add(chatMessage); | ||
this.OutputLastMessage(chatHistory); | ||
|
||
var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 }); | ||
Console.WriteLine(reply); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 2 additions & 3 deletions
5
dotnet/src/Connectors/Connectors.MistralAI/Client/ImageUrlChunk.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,11 +1,10 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Text.Json.Serialization; | ||
|
||
namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client; | ||
internal class ImageUrlChunk(Uri imageUrl) : ContentChunk(ContentChunkType.ImageUrl) | ||
internal class ImageUrlChunk(string imageUrl) : ContentChunk(ContentChunkType.ImageUrl) | ||
{ | ||
[JsonPropertyName("image_url")] | ||
public string ImageUrl { get; set; } = imageUrl.ToString(); | ||
public string ImageUrl { get; set; } = imageUrl; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters