Feat/inline audio data

.gitignore

@@ -1,4 +1,6 @@
 node_modules
 dist
 .env
-.turbo
+.turbo
+.idea
+*.iml

examples/basic/package.json

@@ -27,8 +27,8 @@
     "@formula-monks/kurt-cache": "workspace:*",
     "@formula-monks/kurt-open-ai": "workspace:*",
     "@formula-monks/kurt-vertex-ai": "workspace:*",
-    "@google-cloud/vertexai": "1.1.0",
-    "openai": "^4.76.0",
+    "@google-cloud/vertexai": "1.9.3",
+    "openai": "4.85.1",
     "zod": "^3.23.8"
   }
 }

packages/kurt-cache/src/KurtCache.ts

@@ -319,6 +319,8 @@ function hashMessages(digest: Hash, messages: KurtMessage[]): Hash {
     mayHash(digest, "text", m.text)
     mayHash(digest, "imageDataMimeType", m.imageData?.mimeType)
     mayHash(digest, "imageDataBase64Data", m.imageData?.base64Data)
+    mayHash(digest, "inlineDataMimeType", m.inlineData?.mimeType)
+    mayHash(digest, "inlineDataBase64Data", m.inlineData?.base64Data)
     if (m.toolCall) {
       mayHash(digest, "toolName", m.toolCall.name)
       mayHash(digest, "toolArgs", JSON.stringify(m.toolCall.args))

packages/kurt-open-ai/package.json

@@ -28,7 +28,7 @@
   },
   "dependencies": {
     "@formula-monks/kurt": "^1.4.0",
-    "openai": "4.76.0",
+    "openai": "4.85.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.3"
   },

packages/kurt-open-ai/spec/generateNaturalLanguage.spec.ts

@@ -50,7 +50,7 @@ describe("KurtOpenAI generateNaturalLanguage", () => {
     )
   })
 
-  test("describes a base64-encoded image", async () => {
+  test("describes a base64-encoded image (imageData)", async () => {
     const result = await snapshotAndMock("gpt-4o-2024-05-13", (kurt) =>
       kurt.generateNaturalLanguage({
         prompt: "Describe this emoji, in two words.",
@@ -68,4 +68,48 @@ describe("KurtOpenAI generateNaturalLanguage", () => {
     )
     expect(result.text).toEqual("Heart eyes")
   })
+
+  test("describes a base64-encoded image (inlineData)", async () => {
+    const result = await snapshotAndMock("gpt-4o-2024-05-13", (kurt) =>
+      kurt.generateNaturalLanguage({
+        prompt: "Describe this emoji, in two words.",
+        extraMessages: [
+          {
+            role: "user",
+            inlineData: {
+              mimeType: "image/png",
+              base64Data:
+                "iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII=",
+            },
+          },
+        ],
+      })
+    )
+    expect(result.text).toEqual("Heart eyes")
+  })
+
+  test("throws an error when a message includes inline audio data", async () => {
+    await snapshotAndMockWithError(
+      "gpt-4o-2024-05-13",
+      (kurt) =>
+        kurt.generateNaturalLanguage({
+          prompt: "Transcribe this audio file.",
+          extraMessages: [
+            {
+              role: "user",
+              inlineData: {
+                mimeType: "audio/mpeg",
+                base64Data: "DUMMYDATA",
+              },
+            },
+          ],
+        }),
+      (errorAny) => {
+        expect(errorAny).toBeInstanceOf(Error)
+        expect(errorAny.message).toEqual(
+          "Unsupported image MIME type: audio/mpeg"
+        )
+      }
+    )
+  })
 })

packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml → packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml

@@ -23,45 +23,51 @@ step2RawChunks:
           refusal: null
         logprobs: null
         finish_reason: null
-    system_fingerprint: fp_5796ac6771
+    system_fingerprint: fp_279b0a9ade
     usage: null
   - choices:
       - index: 0
         delta:
           content: Heart
         logprobs: null
         finish_reason: null
-    system_fingerprint: fp_5796ac6771
+    system_fingerprint: fp_279b0a9ade
     usage: null
   - choices:
       - index: 0
         delta:
           content: " eyes"
         logprobs: null
         finish_reason: null
-    system_fingerprint: fp_5796ac6771
+    system_fingerprint: fp_279b0a9ade
     usage: null
   - choices:
       - index: 0
         delta: {}
         logprobs: null
         finish_reason: stop
-    system_fingerprint: fp_5796ac6771
+    system_fingerprint: fp_279b0a9ade
     usage: null
   - choices: []
-    system_fingerprint: fp_5796ac6771
+    system_fingerprint: fp_279b0a9ade
     usage:
-      prompt_tokens: 270
-      completion_tokens: 2
-      total_tokens: 272
+      prompt_tokens: 455
+      completion_tokens: 3
+      total_tokens: 458
+      prompt_tokens_details:
+        cached_tokens: 0
+        audio_tokens: 0
       completion_tokens_details:
         reasoning_tokens: 0
+        audio_tokens: 0
+        accepted_prediction_tokens: 0
+        rejected_prediction_tokens: 0
 step3KurtEvents:
   - chunk: Heart
   - chunk: " eyes"
   - finished: true
     text: Heart eyes
     metadata:
-      totalInputTokens: 270
-      totalOutputTokens: 2
-      systemFingerprint: fp_5796ac6771
+      totalInputTokens: 455
+      totalOutputTokens: 3
+      systemFingerprint: fp_279b0a9ade

packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml

@@ -0,0 +1,73 @@
+step1Request:
+  stream: true
+  stream_options:
+    include_usage: true
+  model: gpt-4o-2024-05-13
+  max_tokens: 4096
+  temperature: 0.5
+  top_p: 0.95
+  messages:
+    - role: user
+      content:
+        - type: text
+          text: Describe this emoji, in two words.
+        - type: image_url
+          image_url:
+            url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII=
+step2RawChunks:
+  - choices:
+      - index: 0
+        delta:
+          role: assistant
+          content: ""
+          refusal: null
+        logprobs: null
+        finish_reason: null
+    system_fingerprint: fp_279b0a9ade
+    usage: null
+  - choices:
+      - index: 0
+        delta:
+          content: Heart
+        logprobs: null
+        finish_reason: null
+    system_fingerprint: fp_279b0a9ade
+    usage: null
+  - choices:
+      - index: 0
+        delta:
+          content: " eyes"
+        logprobs: null
+        finish_reason: null
+    system_fingerprint: fp_279b0a9ade
+    usage: null
+  - choices:
+      - index: 0
+        delta: {}
+        logprobs: null
+        finish_reason: stop
+    system_fingerprint: fp_279b0a9ade
+    usage: null
+  - choices: []
+    system_fingerprint: fp_279b0a9ade
+    usage:
+      prompt_tokens: 455
+      completion_tokens: 3
+      total_tokens: 458
+      prompt_tokens_details:
+        cached_tokens: 0
+        audio_tokens: 0
+      completion_tokens_details:
+        reasoning_tokens: 0
+        audio_tokens: 0
+        accepted_prediction_tokens: 0
+        rejected_prediction_tokens: 0
+step3KurtEvents:
+  - chunk: Heart
+  - chunk: " eyes"
+  - finished: true
+    text: Heart eyes
+    metadata:
+      totalInputTokens: 455
+      totalOutputTokens: 3
+      systemFingerprint: fp_279b0a9ade

packages/kurt-open-ai/src/KurtOpenAI.ts

@@ -247,7 +247,7 @@ function toOpenAIMessages(messages: KurtMessage[]): OpenAIMessage[] {
   }
 
   for (const [messageIndex, message] of messages.entries()) {
-    const { text, toolCall, imageData } = message
+    const { text, toolCall, imageData, inlineData } = message
     if (text) {
       const role = openAIRoleMapping[message.role]
 
@@ -284,8 +284,8 @@ function toOpenAIMessages(messages: KurtMessage[]): OpenAIMessage[] {
         tool_call_id: id,
         content: JSON.stringify(result),
       })
-    } else if (imageData && message.role === "user") {
-      const { mimeType, base64Data } = imageData
+    } else if ((imageData || inlineData) && message.role === "user") {
+      const { mimeType, base64Data } = inlineData ?? imageData
 
       // OpenAI only supports the following MIME types, according to these docs:
       // https://platform.openai.com/docs/guides/vision

packages/kurt-vertex-ai/package.json

@@ -28,7 +28,7 @@
   },
   "dependencies": {
     "@formula-monks/kurt": "^1.4.0",
-    "@google-cloud/vertexai": "1.1.0",
+    "@google-cloud/vertexai": "1.9.3",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.3"
   },

packages/kurt-vertex-ai/spec/generateNaturalLanguage.spec.ts

@@ -60,7 +60,7 @@ describe("KurtVertexAI generateNaturalLanguage", () => {
     )
   })
 
-  test("describes a base64-encoded image", async () => {
+  test("describes a base64-encoded image (imageData)", async () => {
     const result = await snapshotAndMock((kurt) =>
       kurt.generateNaturalLanguage({
         prompt: "Describe this emoji, in two words.",
@@ -78,4 +78,23 @@ describe("KurtVertexAI generateNaturalLanguage", () => {
     )
     expect(result.text).toEqual("Lovestruck smile \n")
   })
+
+  test("describes a base64-encoded image (inlineData)", async () => {
+    const result = await snapshotAndMock((kurt) =>
+      kurt.generateNaturalLanguage({
+        prompt: "Describe this emoji, in two words.",
+        extraMessages: [
+          {
+            role: "user",
+            inlineData: {
+              mimeType: "image/png",
+              base64Data:
+                "iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII=",
+            },
+          },
+        ],
+      })
+    )
+    expect(result.text).toEqual("Lovestruck smile \n")
+  })
 })

packages/kurt-vertex-ai/spec/generateStructuredData.spec.ts

@@ -1,10 +1,11 @@
-import { describe, test, expect } from "@jest/globals"
+import { describe, expect, test } from "@jest/globals"
 import { z } from "zod"
 import { snapshotAndMock, snapshotAndMockWithError } from "./snapshots"
 import {
   KurtCapabilityError,
   KurtResultValidateError,
 } from "@formula-monks/kurt"
+import { promises as fs } from "node:fs"
 
 describe("KurtVertexAI generateStructuredData", () => {
   test("says hello (response format 1)", async () => {
@@ -108,4 +109,32 @@ describe("KurtVertexAI generateStructuredData", () => {
       }
     )
   })
+
+  test("transcribes a base64-encoded audio", async () => {
+    const base64Data = await fs.readFile("spec/data/HelloWorld.mp3", {
+      encoding: "base64",
+    })
+    const result = await snapshotAndMock((kurt) =>
+      kurt.generateStructuredData({
+        prompt: "Transcribe this audio file.",
+        extraMessages: [
+          {
+            role: "user",
+            inlineData: {
+              mimeType: "audio/mpeg",
+              base64Data,
+            },
+          },
+        ],
+        schema: z
+          .object({
+            transcription: z
+              .string()
+              .describe("The transcription of the audio"),
+          })
+          .describe("Result of transcribing an audio file"),
+      })
+    )
+    expect(result.data).toEqual({ transcription: "Hello world" })
+  })
 })