Skip to content

feat: Add audio type according to 2025-03-26 spec #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 32 additions & 9 deletions src/commonMain/kotlin/io/modelcontextprotocol/kotlin/sdk/types.kt
Original file line number Diff line number Diff line change
Expand Up @@ -884,10 +884,10 @@ public sealed interface PromptMessageContent {
}

/**
* Represents prompt message content that is either text or an image.
* Represents prompt message content that is either text, image or audio.
*/
@Serializable(with = PromptMessageContentTextOrImagePolymorphicSerializer::class)
public sealed interface PromptMessageContentTextOrImage : PromptMessageContent
@Serializable(with = PromptMessageContentMultimodalPolymorphicSerializer::class)
public sealed interface PromptMessageContentMultimodal : PromptMessageContent

/**
* Text provided to or from an LLM.
Expand All @@ -898,7 +898,7 @@ public data class TextContent(
* The text content of the message.
*/
val text: String? = null,
) : PromptMessageContentTextOrImage {
) : PromptMessageContentMultimodal {
override val type: String = TYPE

public companion object {
Expand All @@ -920,7 +920,7 @@ public data class ImageContent(
* The MIME type of the image. Different providers may support different image types.
*/
val mimeType: String,
) : PromptMessageContentTextOrImage {
) : PromptMessageContentMultimodal {
override val type: String = TYPE

public companion object {
Expand All @@ -929,12 +929,35 @@ public data class ImageContent(
}

/**
* An image provided to or from an LLM.
* Audio provided to or from an LLM.
*/
@Serializable
public data class AudioContent(
/**
* The base64-encoded audio data.
*/
val data: String,

/**
* The MIME type of the audio. Different providers may support different audio types.
*/
val mimeType: String,
) : PromptMessageContentMultimodal {
override val type: String = TYPE

public companion object {
public const val TYPE: String = "audio"
}
}


/**
* Unknown content provided to or from an LLM.
*/
@Serializable
public data class UnknownContent(
override val type: String,
) : PromptMessageContentTextOrImage
) : PromptMessageContentMultimodal

/**
* The contents of a resource, embedded into a prompt or tool call result.
Expand Down Expand Up @@ -1204,7 +1227,7 @@ public class ModelPreferences(
@Serializable
public data class SamplingMessage(
val role: Role,
val content: PromptMessageContentTextOrImage,
val content: PromptMessageContentMultimodal,
)

/**
Expand Down Expand Up @@ -1286,7 +1309,7 @@ public data class CreateMessageResult(
*/
val stopReason: StopReason? = null,
val role: Role,
val content: PromptMessageContentTextOrImage,
val content: PromptMessageContentMultimodal,
override val _meta: JsonObject = EmptyJsonObject,
) : ClientResult

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,19 @@ internal object PromptMessageContentPolymorphicSerializer :
ImageContent.TYPE -> ImageContent.serializer()
TextContent.TYPE -> TextContent.serializer()
EmbeddedResource.TYPE -> EmbeddedResource.serializer()
AudioContent.TYPE -> AudioContent.serializer()
else -> UnknownContent.serializer()
}
}
}

internal object PromptMessageContentTextOrImagePolymorphicSerializer :
JsonContentPolymorphicSerializer<PromptMessageContentTextOrImage>(PromptMessageContentTextOrImage::class) {
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<PromptMessageContentTextOrImage> {
internal object PromptMessageContentMultimodalPolymorphicSerializer :
JsonContentPolymorphicSerializer<PromptMessageContentMultimodal>(PromptMessageContentMultimodal::class) {
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<PromptMessageContentMultimodal> {
return when (element.jsonObject.getValue("type").jsonPrimitive.content) {
ImageContent.TYPE -> ImageContent.serializer()
TextContent.TYPE -> TextContent.serializer()
AudioContent.TYPE -> AudioContent.serializer()
else -> UnknownContent.serializer()
}
}
Expand Down
34 changes: 34 additions & 0 deletions src/commonTest/kotlin/AudioContentSerializationTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package io.modelcontextprotocol.kotlin.sdk

import io.kotest.assertions.json.shouldEqualJson
import io.modelcontextprotocol.kotlin.sdk.shared.McpJson
import kotlinx.serialization.encodeToString
import kotlin.test.Test
import kotlin.test.assertEquals

class AudioContentSerializationTest {

private val audioContentJson = """
{
"data": "base64-encoded-audio-data",
"mimeType": "audio/wav",
"type": "audio"
}
""".trimIndent()

private val audioContent = AudioContent(
data = "base64-encoded-audio-data",
mimeType = "audio/wav"
)

@Test
fun `should serialize audio content`() {
McpJson.encodeToString(audioContent) shouldEqualJson audioContentJson
}

@Test
fun `should deserialize audio content`() {
val content = McpJson.decodeFromString<AudioContent>(audioContentJson)
assertEquals(expected = audioContent, actual = content)
}
}