diff --git a/.gitignore b/.gitignore index bdec31e..ac7d3d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules dist .env -.turbo \ No newline at end of file +.turbo +.idea +*.iml diff --git a/examples/basic/package.json b/examples/basic/package.json index 65a48ea..05cab3c 100644 --- a/examples/basic/package.json +++ b/examples/basic/package.json @@ -27,8 +27,8 @@ "@formula-monks/kurt-cache": "workspace:*", "@formula-monks/kurt-open-ai": "workspace:*", "@formula-monks/kurt-vertex-ai": "workspace:*", - "@google-cloud/vertexai": "1.1.0", - "openai": "^4.76.0", + "@google-cloud/vertexai": "1.9.3", + "openai": "4.85.1", "zod": "^3.23.8" } } diff --git a/packages/kurt-cache/src/KurtCache.ts b/packages/kurt-cache/src/KurtCache.ts index 29ed8bc..92d7d79 100644 --- a/packages/kurt-cache/src/KurtCache.ts +++ b/packages/kurt-cache/src/KurtCache.ts @@ -319,6 +319,8 @@ function hashMessages(digest: Hash, messages: KurtMessage[]): Hash { mayHash(digest, "text", m.text) mayHash(digest, "imageDataMimeType", m.imageData?.mimeType) mayHash(digest, "imageDataBase64Data", m.imageData?.base64Data) + mayHash(digest, "inlineDataMimeType", m.inlineData?.mimeType) + mayHash(digest, "inlineDataBase64Data", m.inlineData?.base64Data) if (m.toolCall) { mayHash(digest, "toolName", m.toolCall.name) mayHash(digest, "toolArgs", JSON.stringify(m.toolCall.args)) diff --git a/packages/kurt-open-ai/package.json b/packages/kurt-open-ai/package.json index 77066f7..510f8d5 100644 --- a/packages/kurt-open-ai/package.json +++ b/packages/kurt-open-ai/package.json @@ -28,7 +28,7 @@ }, "dependencies": { "@formula-monks/kurt": "^1.4.0", - "openai": "4.76.0", + "openai": "4.85.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.3" }, diff --git a/packages/kurt-open-ai/spec/generateNaturalLanguage.spec.ts b/packages/kurt-open-ai/spec/generateNaturalLanguage.spec.ts index 3b76366..9c27ece 100644 --- a/packages/kurt-open-ai/spec/generateNaturalLanguage.spec.ts +++ b/packages/kurt-open-ai/spec/generateNaturalLanguage.spec.ts @@ -50,7 +50,7 @@ describe("KurtOpenAI generateNaturalLanguage", () => { ) }) - test("describes a base64-encoded image", async () => { + test("describes a base64-encoded image (imageData)", async () => { const result = await snapshotAndMock("gpt-4o-2024-05-13", (kurt) => kurt.generateNaturalLanguage({ prompt: "Describe this emoji, in two words.", @@ -68,4 +68,48 @@ describe("KurtOpenAI generateNaturalLanguage", () => { ) expect(result.text).toEqual("Heart eyes") }) + + test("describes a base64-encoded image (inlineData)", async () => { + const result = await snapshotAndMock("gpt-4o-2024-05-13", (kurt) => + kurt.generateNaturalLanguage({ + prompt: "Describe this emoji, in two words.", + extraMessages: [ + { + role: "user", + inlineData: { + mimeType: "image/png", + base64Data: + "iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII=", + }, + }, + ], + }) + ) + expect(result.text).toEqual("Heart eyes") + }) + + test("throws an error when a message includes inline audio data", async () => { + await snapshotAndMockWithError( + "gpt-4o-2024-05-13", + (kurt) => + kurt.generateNaturalLanguage({ + prompt: "Transcribe this audio file.", + extraMessages: [ + { + role: "user", + inlineData: { + mimeType: "audio/mpeg", + base64Data: "DUMMYDATA", + }, + }, + ], + }), + (errorAny) => { + expect(errorAny).toBeInstanceOf(Error) + expect(errorAny.message).toEqual( + "Unsupported image MIME type: audio/mpeg" + ) + } + ) + }) }) diff --git a/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml b/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml similarity index 81% rename from packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml rename to packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml index 93c043a..69c97ef 100644 --- a/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml +++ b/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml @@ -23,7 +23,7 @@ step2RawChunks: refusal: null logprobs: null finish_reason: null - system_fingerprint: fp_5796ac6771 + system_fingerprint: fp_279b0a9ade usage: null - choices: - index: 0 @@ -31,7 +31,7 @@ step2RawChunks: content: Heart logprobs: null finish_reason: null - system_fingerprint: fp_5796ac6771 + system_fingerprint: fp_279b0a9ade usage: null - choices: - index: 0 @@ -39,29 +39,35 @@ step2RawChunks: content: " eyes" logprobs: null finish_reason: null - system_fingerprint: fp_5796ac6771 + system_fingerprint: fp_279b0a9ade usage: null - choices: - index: 0 delta: {} logprobs: null finish_reason: stop - system_fingerprint: fp_5796ac6771 + system_fingerprint: fp_279b0a9ade usage: null - choices: [] - system_fingerprint: fp_5796ac6771 + system_fingerprint: fp_279b0a9ade usage: - prompt_tokens: 270 - completion_tokens: 2 - total_tokens: 272 + prompt_tokens: 455 + completion_tokens: 3 + total_tokens: 458 + prompt_tokens_details: + cached_tokens: 0 + audio_tokens: 0 completion_tokens_details: reasoning_tokens: 0 + audio_tokens: 0 + accepted_prediction_tokens: 0 + rejected_prediction_tokens: 0 step3KurtEvents: - chunk: Heart - chunk: " eyes" - finished: true text: Heart eyes metadata: - totalInputTokens: 270 - totalOutputTokens: 2 - systemFingerprint: fp_5796ac6771 + totalInputTokens: 455 + totalOutputTokens: 3 + systemFingerprint: fp_279b0a9ade diff --git a/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml b/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml new file mode 100644 index 0000000..69c97ef --- /dev/null +++ b/packages/kurt-open-ai/spec/snapshots/KurtOpenAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml @@ -0,0 +1,73 @@ +step1Request: + stream: true + stream_options: + include_usage: true + model: gpt-4o-2024-05-13 + max_tokens: 4096 + temperature: 0.5 + top_p: 0.95 + messages: + - role: user + content: + - type: text + text: Describe this emoji, in two words. + - type: image_url + image_url: + url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII= +step2RawChunks: + - choices: + - index: 0 + delta: + role: assistant + content: "" + refusal: null + logprobs: null + finish_reason: null + system_fingerprint: fp_279b0a9ade + usage: null + - choices: + - index: 0 + delta: + content: Heart + logprobs: null + finish_reason: null + system_fingerprint: fp_279b0a9ade + usage: null + - choices: + - index: 0 + delta: + content: " eyes" + logprobs: null + finish_reason: null + system_fingerprint: fp_279b0a9ade + usage: null + - choices: + - index: 0 + delta: {} + logprobs: null + finish_reason: stop + system_fingerprint: fp_279b0a9ade + usage: null + - choices: [] + system_fingerprint: fp_279b0a9ade + usage: + prompt_tokens: 455 + completion_tokens: 3 + total_tokens: 458 + prompt_tokens_details: + cached_tokens: 0 + audio_tokens: 0 + completion_tokens_details: + reasoning_tokens: 0 + audio_tokens: 0 + accepted_prediction_tokens: 0 + rejected_prediction_tokens: 0 +step3KurtEvents: + - chunk: Heart + - chunk: " eyes" + - finished: true + text: Heart eyes + metadata: + totalInputTokens: 455 + totalOutputTokens: 3 + systemFingerprint: fp_279b0a9ade diff --git a/packages/kurt-open-ai/src/KurtOpenAI.ts b/packages/kurt-open-ai/src/KurtOpenAI.ts index ba43d8b..ad256e3 100644 --- a/packages/kurt-open-ai/src/KurtOpenAI.ts +++ b/packages/kurt-open-ai/src/KurtOpenAI.ts @@ -247,7 +247,7 @@ function toOpenAIMessages(messages: KurtMessage[]): OpenAIMessage[] { } for (const [messageIndex, message] of messages.entries()) { - const { text, toolCall, imageData } = message + const { text, toolCall, imageData, inlineData } = message if (text) { const role = openAIRoleMapping[message.role] @@ -284,8 +284,8 @@ function toOpenAIMessages(messages: KurtMessage[]): OpenAIMessage[] { tool_call_id: id, content: JSON.stringify(result), }) - } else if (imageData && message.role === "user") { - const { mimeType, base64Data } = imageData + } else if ((imageData || inlineData) && message.role === "user") { + const { mimeType, base64Data } = inlineData ?? imageData // OpenAI only supports the following MIME types, according to these docs: // https://platform.openai.com/docs/guides/vision diff --git a/packages/kurt-vertex-ai/package.json b/packages/kurt-vertex-ai/package.json index 26d9b76..3fb3e10 100644 --- a/packages/kurt-vertex-ai/package.json +++ b/packages/kurt-vertex-ai/package.json @@ -28,7 +28,7 @@ }, "dependencies": { "@formula-monks/kurt": "^1.4.0", - "@google-cloud/vertexai": "1.1.0", + "@google-cloud/vertexai": "1.9.3", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.3" }, diff --git a/packages/kurt-vertex-ai/spec/data/HelloWorld.mp3 b/packages/kurt-vertex-ai/spec/data/HelloWorld.mp3 new file mode 100644 index 0000000..c4f149a Binary files /dev/null and b/packages/kurt-vertex-ai/spec/data/HelloWorld.mp3 differ diff --git a/packages/kurt-vertex-ai/spec/generateNaturalLanguage.spec.ts b/packages/kurt-vertex-ai/spec/generateNaturalLanguage.spec.ts index 1864734..a9cdb81 100644 --- a/packages/kurt-vertex-ai/spec/generateNaturalLanguage.spec.ts +++ b/packages/kurt-vertex-ai/spec/generateNaturalLanguage.spec.ts @@ -60,7 +60,7 @@ describe("KurtVertexAI generateNaturalLanguage", () => { ) }) - test("describes a base64-encoded image", async () => { + test("describes a base64-encoded image (imageData)", async () => { const result = await snapshotAndMock((kurt) => kurt.generateNaturalLanguage({ prompt: "Describe this emoji, in two words.", @@ -78,4 +78,23 @@ describe("KurtVertexAI generateNaturalLanguage", () => { ) expect(result.text).toEqual("Lovestruck smile \n") }) + + test("describes a base64-encoded image (inlineData)", async () => { + const result = await snapshotAndMock((kurt) => + kurt.generateNaturalLanguage({ + prompt: "Describe this emoji, in two words.", + extraMessages: [ + { + role: "user", + inlineData: { + mimeType: "image/png", + base64Data: + "iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII=", + }, + }, + ], + }) + ) + expect(result.text).toEqual("Lovestruck smile \n") + }) }) diff --git a/packages/kurt-vertex-ai/spec/generateStructuredData.spec.ts b/packages/kurt-vertex-ai/spec/generateStructuredData.spec.ts index 82d9c5c..b24a7a0 100644 --- a/packages/kurt-vertex-ai/spec/generateStructuredData.spec.ts +++ b/packages/kurt-vertex-ai/spec/generateStructuredData.spec.ts @@ -1,10 +1,11 @@ -import { describe, test, expect } from "@jest/globals" +import { describe, expect, test } from "@jest/globals" import { z } from "zod" import { snapshotAndMock, snapshotAndMockWithError } from "./snapshots" import { KurtCapabilityError, KurtResultValidateError, } from "@formula-monks/kurt" +import { promises as fs } from "node:fs" describe("KurtVertexAI generateStructuredData", () => { test("says hello (response format 1)", async () => { @@ -108,4 +109,32 @@ describe("KurtVertexAI generateStructuredData", () => { } ) }) + + test("transcribes a base64-encoded audio", async () => { + const base64Data = await fs.readFile("spec/data/HelloWorld.mp3", { + encoding: "base64", + }) + const result = await snapshotAndMock((kurt) => + kurt.generateStructuredData({ + prompt: "Transcribe this audio file.", + extraMessages: [ + { + role: "user", + inlineData: { + mimeType: "audio/mpeg", + base64Data, + }, + }, + ], + schema: z + .object({ + transcription: z + .string() + .describe("The transcription of the audio"), + }) + .describe("Result of transcribing an audio file"), + }) + ) + expect(result.data).toEqual({ transcription: "Hello world" }) + }) }) diff --git a/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml b/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml similarity index 100% rename from packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image.yaml rename to packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image_(imageData).yaml diff --git a/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml b/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml new file mode 100644 index 0000000..6f31543 --- /dev/null +++ b/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateNaturalLanguage_describes_a_base64-encoded_image_(inlineData).yaml @@ -0,0 +1,63 @@ +step1Request: + generationConfig: + maxOutputTokens: 4096 + temperature: 0.5 + topP: 0.95 + contents: + - role: user + parts: + - text: Describe this emoji, in two words. + - role: user + parts: + - inlineData: + mimeType: image/png + data: iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII= +step2RawChunks: + - candidates: + - content: + role: model + parts: + - text: Loves + index: 0 + usageMetadata: + promptTokenCount: 266 + candidatesTokenCount: 1 + totalTokenCount: 267 + promptTokensDetails: + - modality: TEXT + tokenCount: 8 + - modality: IMAGE + tokenCount: 258 + candidatesTokensDetails: + - modality: TEXT + tokenCount: 1 + - candidates: + - content: + role: model + parts: + - text: | + truck smile + finishReason: STOP + index: 0 + usageMetadata: + promptTokenCount: 266 + candidatesTokenCount: 5 + totalTokenCount: 271 + promptTokensDetails: + - modality: TEXT + tokenCount: 8 + - modality: IMAGE + tokenCount: 258 + candidatesTokensDetails: + - modality: TEXT + tokenCount: 5 +step3KurtEvents: + - chunk: Loves + - chunk: | + truck smile + - finished: true + text: | + Lovestruck smile + metadata: + totalInputTokens: 266 + totalOutputTokens: 5 diff --git a/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateStructuredData_transcribes_a_base64-encoded_audio.yaml b/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateStructuredData_transcribes_a_base64-encoded_audio.yaml new file mode 100644 index 0000000..a3e478e --- /dev/null +++ b/packages/kurt-vertex-ai/spec/snapshots/KurtVertexAI_generateStructuredData_transcribes_a_base64-encoded_audio.yaml @@ -0,0 +1,63 @@ +step1Request: + generationConfig: + maxOutputTokens: 4096 + temperature: 0.5 + topP: 0.95 + contents: + - role: user + parts: + - text: Transcribe this audio file. + - role: user + parts: + - inlineData: + mimeType: audio/mpeg + data: SUQzBAAAAAAAIlRTU0UAAAAOAAADTGF2ZjYxLjEuMTAwAAAAAAAAAAAAAAD/+2TAAAAAAAAAAAAAAAAAAAAAAABJbmZvAAAADwAAADYAADOQAAkNDRISFxcbGyAgJSkpLi4zMzc3PDxBQUVKSk9PU1NYWF1dYWFma2tvb3R0eXl9fYKGhouLkJCUlJmZnp6ip6esrLCwtbW6ur6+w8jIzMzR0dbW2trf3+To6O3t8vL29vv7/wAAAABMYXZjNjEuMy4AAAAAAAAAAAAAAAAkBmAAAAAAAAAzkOFMiDoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/+2TEAAPLjSTwAYBmAYgkX8AxDwBERAghE396fX+u7nYiIiIIA33dwN3CAAAiFAgh3cOBgb7oiJAgAhfXP66JTiflXNEy3Dv+7vwIAInoLfdw4tz0RCiAAg+m7/RCHIYQsPow+U4gdXLg/4AD/hju8mL0JAaWoxD8MEalriLeGbOOQKPA/jFTv7AiXQIp7IieqS2zMh81BGJEFhypdMVOcpBKlrpR5CyzTdYWcY0yNDXHrEC8GMNaKi8gbkBryohlj7lsELjgARZFIPv2w9Smh+xV6hzm6KMx8I8dTatSEAlZsRspPOIIhyuCjXWUIRD/+2TEDgPMOSj+AYR8AWqhIAAhijhDaq65o1RU3EPUqu6G5ox49pEv3zpbEpIhvEjvqeRKSfJxvi5EiGDGm3NNGDDRdYwilFQqbaCA2sqm5CEaqXNYuxABWHpgdyWLVmqxYXWySM/7nogLv/nQTkrimZqkvTPBHJst1yXIzJGPEkFp9tYQlcHZPoVyI2wM/DJLvFQktMImCobgi4iXn0J2mAGTaKRVotYdLPYNiw9VwvOqKk39tQNeq8ORmJSZkBSQG28iqFkMmPGCxENoYU1M4gZHNSQYmYyZIXv9ypFBzIZFJzP6KBEWkI/pLJzUyJX/+2TEHQGL3Sr+AYRuQWAtoBQwj1hqNEOlo+f+/aILDRdb8y0RNc4BW0pIm5hzMqUYiSiSlJNbZPVSgFvJvK9u1O1ZsZTf3LhraJk9UIsV8zQScaZHJgbrThGFWCkwkYyMFhQjb7uNZBjkfShQHkmeUOZneZNSlO6eUs+9+qv/MufjkzpWHFqU0qlaFaE+fmHa0h7ak6MioVUM51VSZTpzqcQze+wXub09uemkQlYlO9XlMsiOmeLU7+tj0v/zN69UpKTPTc7mrGFN2sCBjC0ISo+0WWa/9w9Igo91W7y7otDx71zxnwdzYif6ta29bTn/+2TELwPL0Qb+AYRqCXilX8AwjMgmZwUQhcngq7VqcORl8qQWLXTySGykZw5rCaP7kqnS/q2U0/Bo9RyLQtW9uqcjTi+A/c6VyORNARTssz65PSXBgwmNzjiwooS4EImZMmswkrbIApTU4a1iw9WgSFmMTR6C4MDBS5fDwuwKhEJqG5m4JoXBbypFRo70Lf6UiwpFBEhzpw5C5lzfahF8E0MhWK+EPgfjJfJ/nXBu2HIqL5yh6Mb1evKB+/f4hv7v47Uhjhej9jUb+BMx3zDu3x9yuceAnEIO8sbEdEWe8IlBwQY6kNBkj0pq98Q1fE//+2TEPgAUVS0CFPeAApakqbc1kALpW94d9018T53m/x/IkkOLi6BdyhjmT9C6kuF33vvKF1bz+FzfrHb2M1rxenU6jQ2HUisARl0MTQ9WajpGkU0LNmPLq4fF1W/Fg5aBXSqSANZwekbikpaG4krAw4JHMiMrr+Gm5N+veUGwokGWnYa8kcvQPJbliTvQXMDgFhORuMUV2a3Vlt+xIKTB037p6e1DbkSzD9b13/p5euiQZxepM2ZiDr9rDOav6yxmd2L923TSyns563rtfWf//////5SyksWLfP/4vdjd+xfVkhIIAAJRSgBKANGMiM3/+2TEBwANjRVfvJSAEbkjLPWUmXpROqToAIFBuqYWUQNpx++1L9sooCA8obZGBIygiUJyCox3NQUlep5IpNIx22oo20raQEAotC0JRtoZAGjSIWpoI3qUM/z+5+pY23WX6zE2yPgugJjorZfJdcufkSUxfubRBSAJBToND0tey9NNK9nbzOQyhXiPTxk7ZqRPFIQhwKhgVoNEpoFQegglDEiebXzO4gRtBkLkCqNvb+e2ZnNYzoT8pPLKIYUEHEgI/VHikHjH9EFM+fx+/Qv//bJmE2gLMKJnrTXOGWysaMXirnIiIAAAAAAaAInjkDn/+2TEBwCNmLNh7j0pibQgqvW2LemOlJw1gCRoFGBQm0tTd3XknTRyoa0oefR5tTS2xIec4pFpK3YURQqGJJmMdpDN1qLkxFfhvvPl50UYwYNuEm/g3JlWMyizg0BpCyNJKhMvsoOf+rxKTkIZujoP96dXyHAAAAACwTuM/GDlEky9Mh1GN+1rA4AaiUBzlJ9NNmVqu3DhUEUw2rtnYPbvHU+QwMolhVWL5httUrS52vXy//+Jxts3afJhHNAwI5aetjHF4+l6raaynN/9v//7olOHVdteqtfUQ6WVCbOsjpoAAAGgKAzCWYdFlKbrYDD/+2TEB4PM1QdSbSRRwcScKQGnmjkGrIGDq7QWR2S/QOZEiKz1/YvGJhNaXq6aWFQqBqQEjaJ8kUmpdf6opf/////l4lSVUUMhcWPGUJDKrsSiEKjLCA+0ZZ//9/+5AgwDsFZq+tgFBbBQph9K+DpeDLgDelTAqy6pOTLgmAGEpcWOlQw3co6tAp2xzdDIKesWNb01qFAXB5K1rHU+U0HEaPqFZy2ynZaan///l766FJlvJm4K5xVj+RlWGSM2vUedrKoWBta2ln3vX/7N//7u3QARi0W6iF1f/LQjU+oA4ABgmoeWAaQeZ1MIdJnwJqj/+2TECQKNYNtGzSVvybKa6BXMsJs86CBYwRTxLCV/C8KqUPPBPGq2W+V1CTLAifIhPj6jWaz87bNCWLt//r1eIWASUSB1oFSZAKgyHzG0okfEwJjB1EMPxtbf/+md0ecepdZW2e9QxFrRg9Xi/RACwLMMStDmGWkw4gRJ1GGgv43wCoyukq3XiY5ZUAuPDC1prTmGaL03WlWXKntKZb7fXJaPoVvTMzM1sdNFs/Dcnj6PryERC+eHZ0sfYOQ5QB5NR1N1X7M7+/TfpViF+lecdbm/0n/ImQDFmeUAAHTABAGgBrEwas4cN+ShxdOttrL/+2TECoMMPKtFLemBQaebJ8GzMiEKGEJm2luy1UjgH5dvDO572Vp3Yux1YZG65chrdtR6fnoK0mZmc2rT60wIcJJsHyUwMkOjiF1cX4vE5IWFhUP+QHqHoPG/YCh46QGNHrsOxqjTBM29yoxsOYC7w6LhQnLoEAHNjIDDSz7E9AFqbPLlIukKiRECnFEiiSnRpz68CbN17TP3llcFJwTx8Xh6YpVZawsq6I5xYJR+UGjEIl59+ds9tq2PII1r2Jjjmb/j9lLAO5caLDUAAAAAOELAYBgBJbVVMDWLISmpuUXKoS+RKRcsWZhQhK4N6iP/+2TEEgIL/NNJzeUjAXAYqLGjLfHpJ7m+Ff3dpwvfk8zYuZGUkpT//hqXf4haAHkYokarwnL3Nl0RSVSSbl///6x7CyR9FBcw6x84yxaRpJIKpBgEE6j2VgeWCd4NNhEZSVMFRqhgUJwyxeVtZk0Yvn4ammBBF7dIS+QfU+2z+y0ANIf0Ie2566iHNkkEhMTm+eWZWbr1LDZzk4j7tj/+0459Gf//cEDs5LtGNgAAACN6iRspQCDBnBSAIHVEFCQhtx5a0wYCWasiWNo9UdsLFY84DUHYVQZLAL7UTdJDc27FPDjSGb4fK9VLsccfle//+2TEIYAP5Q9b7LDY6YYWLD2HoTj+7kzNbQ3v78V0gKKUJAJC1K8r723BzuJFCYJCJhRHWuT2SU6iEMEZpkvf/+QeUZqh4wkJEFpTkGKILgcAABABpSTUJzAWVrpQI7aKiGhItOm5oUzAMIiNwDy6MqmpU4okbj1xPCdJJDCDHAQRAksisC5oux4hGlVfr///MMLOcro83vMjgaiIJ2B4OChSzmIBA8NgJwRLdFD1YGAwrQAAYAAANAayO+hwoJBXqAYXpAYjSWHjpCdaO49w9N1LHQ+8Hzam+seSkeRF1JIHUyKFqpYpzNf//xMmVUn/+2TEHoJL1Q1VTL0NAWaV6m2Xjammv6dJacFBagXDDgbCpdL339rNN8NDyvcf/Xe988z7McVYAgJGQt2HJhZdM8zikrjbBkKqTPgfGjuHyM5YsJdtc4UCFEl+cXtM9gKQpznZ0OWGWVwxYIGR8vL/+HlKH0kcObcLchER8SJEP/1UhF+DJQ7volWByUqT40lVAABgAAAwCwKLkY7OSJoR9FApgOixbM2Uh4CSRAbK2W24Al8kwt7x3hrm9TUTjrSFlOXyNyukpZ+msALLbT+iqh2QXZYJgRkWgWFfbYEhOL45dJVQAPirCiTWXfmJYgn/+2TEL4IL4LNTTIk4gXWhKqmVijlDQEwDgJwsBk480RTkLwmQ8JM+h1WoI/l1TrnTUjrSfO8juPxTtE5BmFYBgfiw0J9LUfSOO217qbTf3so+27WZW/t1SxB5LgrgWHSD1L76nR7WWjqxp/xSmtoq/BMqAAG0CQB4CwSjk2ywZFWQuE4MMo5WQN4KpAaOBophWoUnYZr8Gf9uSvF5DABYkgWHtW2VBa1MJMTX9S01T3PEJXFW5geGCgnC4aHHNX/3CrNVUXM1Ks/VDyPIFSgCEIWAFgKAUBKnByYlKkgIRpjhn17Cw1eRm2aojFkpE/X/+2TEPoILxOlXTLENAXaaKimnoaAzXNd1nFHeL/fvKsNQ4nqdHCWnsqlM6mv575+GpVkUg+XYcM3qbFLDsXD4EAqUUL9f+0XtFmEiZp19DBnJzyoAANRYDB9hRuaIEmCA7itp92DPwUWdowFaLgOptGCstrQ2rN4mLeFuDmkZTPSeiguSkI5pytQlcrMz59vP/631fqN1BpV9a/DYLCirAMLMiiLzFVgPCYaYtVAZYJsnxz1kVgCESoKCRfYFlhjEqHo28ZwZgaqUy7FFlpcZjt67O/R7s4W/3+u3O0GFOxyweNk9OW7SxI5S0c1//zv/+2TETgJMDLNNLT0rQXceKXWliuGe07sqFjh+LUNhCVOh6HcFz5WZGLa/+26OCf/6HQI9KX3K1QAAABeTEyMAwChFDQLeohBPdAUZEm2U0pZToiKM3dmeqw6/DNTR8h/h4+ulpRoKDbCBEBxMSSxamVGYZ5d/52ZyCyO6GU5Yog4YADgGNDbzN/mctRx2/3mFCioifHEKXB1ASAg3cUjjPhMEt5xqACQs0csW0cwMtfBxNARMHJfu1RbsrfWTMLfz59PqDSgYNYYKPxsw3/f+zJKIiqpYYJiSYaYDoFpUFg6AZGqjUi39y5qzmO+86iX/+2TEXAIL+PVN7SSvSXIeqHG3nWiHFHKqVQAC4ERn4jvjfQUQUZ0Q4YMGj2AKCBzYjYfpxWX1Ix2L3gbwqxb+01QqoJuLLdDGS5V7ev7///5mjbLrFVSquzQCYnAijaP/EV//8G6rbujj+/3dTLHHlTQ2aJJb/YAAJWrCUALAfskuBeuLDyeIGgAwA2CDSs2sgTRMB/OAUP2RYcLBqU69R2O1UXRVFNJmr6QdqqZ87X19zx//+xjUdZzd8EIRAOlSw/XP1//o3BtZ1rrr/+Y0TizEU712CJF6AAAAzQKgAQCgxgyIG1QMYLWJogCmHQf/+2TEawALpPU/DaFvyYeep7W2LSEULkxAAxolt1XK/8VWVKnhjMWzgKNmVHQpoyYEKKcrNNUlxI2tIUn3NqW7RAboLglEhKYPjj0v+hCa7qrFF/+jHHOzXRSqLfgDAAg6UoDBEGDBaADOyY1DZBAmZVimCBJr5sTDaC8RfCGleuAzZmdoPXuZGXrKtCQStaju+5ppFa/1tLqLnj//7HM0XeBYeoIwemesJh0WHfFjIbEoVWIxuaUAauAgYSgQabxIbVAoYhngYlguYdmYYViqCgsETwfYligqSne2NZjQmXw5dsIlFBxk1EocYQDfZ9//+2TEeQBMBQM5rglRiWGU5mndoDCxBY3fDd0lSF8229GnlAOilFEXP/b1KttYFL/5ebzg0mK7LN1oQWsgYdkaam+2aPDGYNFiY4KnAFIMcQEegwpb9CQxtCUOgUcUwvvtfgWuIeCuZhKBQ5VITk+Q7iyWt4lldZI7mfFMnyXLOyKJEjfdfQPcEjgxvfif3bJ8siHTtjPcPTzTXTksvTXktjDkDAxMDCAgQMA5t4uwO6SJDjhoxRSZKGcUUvs3L/256kqxzPMDEDCXHUGDKgjoOCCTfqcndSmZ3pspL34yGLwlc/hd9P5Bpznd8y/8pJD/+2TEigILuNUozuBnCWyVpNndjHGPXoxAhKUQMGkIAwSEJjAoBQMUoIY1hw8gAwHYwxA7BJBdASjkFxsNLfL/vdvUtjgZiSd1ji5LkGDEDycd9bDrRmjO072p77p7H/j4Qyb3/pgVhZMLB07y3hkMjGMzGiPv+f6737+/Gj7n19c9Pj4QDAkhzxP1zURQNUQ8ynPzMgfMjrcxkujiAgMDhswSDS2oNEK8VMVaFePBRt6qZZQmwmznHolqRLNDsOw+GpY0wjglahytfmGYmKUUWjcVrIb3y7VI+WbdzojwrLnTlIWbsVdhYbX2rZebFjX/+2TEmwILyM8gDuBnia4hZSXtGHG1ilty87M5Fel+6df+su65HSjrLjcq8IYbgQccD+Z8kacQjaY6uWYsgyYTh8YDFMciGbFuEUgNcChEHAgoCf9Thdk60yhnrWuZXbdM+8Bz8iiEYyIAC4NzmNMKbsdXb9+/9O7fY2yzQfJh3JukJSUu01RONVz4OTby0O+v07cwyLyH3cUlyRzYVTUAACB8BAAAkZ4wsY/ByajG8YHMuYog8VQAbgachKIgaLgGAoLoyNMmk1KjbtXooxDU3cjUAwtckmeJEkMWG2CZKhdFAzZDRMUct2V/J2//l7v/+2TEo4NQWQ0mLvGBmcqgZgXdGPq5y9x9vV4lvmZ/7ZUNQCH490ocLIu7rTkohRgZ6Xn/mrAcANFu8mYdPuy44GDg6ZD0ZioOGXkqYWeJvYxGGgkX1CJQYSIRq4TOIAhI5Eg20SreKenoc3VdtYAHEYK1F+4ZD0SjjSTB9TmEzWO0N+Z//+d80sIw4LB4FAeAKh+mMASAMHYjgOFhOLDfr/8dFEKJyt//71lkMGnpuVDrE3H5yC+wCgGFG8JA0xG7hAjOTyBELxAgAMDCkB/JmVUGALrQ/Y4phBkkhirLXjhbFw9CQ3pKx7qW5wy/aan/+2TElgJPdQM5LrB1SdkiJ8nMIWBUZlr+vmkiRnQ8aKB0GQmDw17tQ4D5xgwUMSnv/qm5YaSvMsta/45toMdR2lEsACwEwDq6jHCTOGCAeB0ph/veNfCsxEAc7wG1g8lBf9MOafKHoZgZsMDQyOLcVt7ZzWYQ2Yik6VlaY0//+rRZIDtAjBwCYP2LnXk61iU8Qxamp+1YdY6GfWv/44/rtSxMfTQjV5afVZMQFAFMQZvWIBuJChJoJmGCwaxkf0rDRNwYPlSwjS4KgidayxsRBY+tJq7zHpSaSwQG5LfbHZmc3/XdhQVUwwJB4MBVRFH/+2TEioMM2M1GbeUFiZ6hKQ2soLFhgcaImCAuPM3QwszDXdDP/9U1Ky+p2FRMYJAqOgB4BqhnpQJJGHCkLdnJoWqT8JAAWHgVqHSqiCV6wbqpvUKrodrz1eyCQ1kOVEUEJBIXZL2//uRRgwRChMhUCMpkUIoYW5Ys0QGr7+BYRSGO+I//4pIWv5rWvmbGjCjTFQAAKkyAlAf0+VV2FN4kaYaps0J7xAIaGTSJZIBD8rHLAcBIAUMGPbAZT4rvjtLI06J6OZ9bz3/pt//9ZuX9DA4ABbOwabkuwEGQixgYkOhOsGEKZGW//ZyC1RTflsH/+2TEkIIMEQtMbSSwyYch6amhImDIJFABS3gBUCsLgRCJGpxgzxKFG4ClidxQNLgiMBD46jSjQTDoByg4FLqUwilST2vDGu3w+A+zkUFNKyZf9lUggADAIkUBgCGOVCMADUgJTPfaOcPC/8TgEH1YqCh4FUIAaUABYFOC3xCdDKxkiJfcByWZihBMEWHBYwmsLOBGVXIu0kCQE5k/INhSTXupMuHrcYl91YxLmUfv/+Y3klQuKCOJAigKSRLSQaPJcY7nIO+ZlLr/15zi7gPK3uzMHiCScAGbYAJgQ4cMEGFAIObhpbMXiTqJKWjTGYj/+2TEnIIMFQtNTJhRiXQYabWjChimWPzIrBccxUhJpTAFGmCW/s21azEOgyqxmTUyAgCLE/NXuvN2MIf//5BRSQbGSICicUiYLhZqDnR0rAlUOWZccbuvkKeqnuZ/P///3VZjzFPnPf//t1LYxt0AABnKAEgF3HHSjpouOPAgyhGRpGgajMKAVXrEIRpIDCBi0a7X2VQIJMRoguUmdZVL0BgDAFw+9ElWV0LZP//3pxo3HCGoQg5VxLQJw6dFdM5Ov7ZLi04//tJaXKHCBZ4wzrh84NY3gAEgVWpjfQCfkpkyIYa3GO9GRJtdBTFSABT/+2TEqwIMMQlI7SCxmc2iaLW8pKimWDgEUaBCxRjLOUxW4p2gnJoZWeq5NjdaKhhDlDq57uH19//8bGFRcogaKmg/BwhT1GiRLNkQ8GUW9NhmdjjdLJ//eR4mIMDh0YE3+8XQAG3iAoAKAgDRQOIDAENgE0Y+486theosAUKgAQQIPojiq1/AdLYZYRBDAkkks5JeZEFBCjqdvGIlIn/zyPQWMYowVQYJSBHaDVIEYMjDBFDgHCnFXr///6iP+v1qIUizqrr/joyE5QMEcAGwADwf+JsimGEKJOeB7h/MBPId3mEABkCByIkCBsajQqn/+2TErgIMxPNLTRkQkaceaA2lljggfdv1E5j77YhnRMSk9tWg/fMnxbRPfj//pVSLiGKcR1lVJGjghBaHxBRwjHNDf/vFf8/08ffCMo7e//+GhjqaAAAAMIOZEoBccnBwMPIAVGS3AXBwURoQJXF6QqBhUAV4+qIUtl/DtXKN3KIEHXqF1W2HL7n4AxMIc0fvPQPGCOpsM7KuclqRkDsICun/6OU9izJf892TpwRTmZzTA4MDLAaAgLAV3mXREV0MFAYiDTRdE0YMoAJmAIMY4GjKwWBEQ/OrHq5NbfE7PHQVFBdzZ1B8lp8zyjxMTP//+2TEs4IM6Q1G7eUBiZEhqOmWIalFL8nmCOOfZEuJiSAow+A/UaVN//9f+MVpav5j95iTJxCTPGg75gAAAAM4MGEh0AYVnoDAIq0AaBICTDjkdRwEvByy7Cl01TymHKcKIsTyaZ9Rp8FiFRB1OywjCnZdyL/aoYbcAOt2FmWDDBxIKzI/////cE6GSnS8meVY4MCDkGdIlmRgCkAWA4L4PoCMCJZmWAzvGqJmJABwcMXIdSyAiIWg0uhMyfF81//P7pKIkxIhLNwk7zWFd/pP+ZHzRgvCD1MHrMLPchwKhQcxogY2ev////nela//dRX/+2TEuwIMKRFLraRtSX8eqGWmIaH0qLoxFN4iAAAC2LQBAFfDsdAUwkEHBmDCEsM5ASYisHaTlZLBlNPXYdcKdmbJGl/ucVDihAC4oHLnMxgNCyE57v/+TVo5JBrQPHl3xFqtCwOhAJRMwWNQ4f+y8OtlA4dcVMpw6TB2IwGHgGLEg5oHGFaAUVGTQiELaElZMPHhYHyW2drXXThhij8M/WCjTUTomsGo+UYaagq62/C/vzb6KgiHDh8TUVBSGnW+ND4MHg+QeHxEff/71yuVqO9xFCshYkHr94mMVQAACtSEBYGLUHTODQQtgyYyIgr/+2TEyAIL1P1J7RhuyXSd6HmkoXHIohBcEXXbg51AYQoQPRipGapRv+vByd9wEnk73ezEgp3Er/ycZm9JWDWi4MBCKT13b5n9B1IgYv3G/////y85kzDHm+qz2932eVYjX8TdviUgGAwcFE5o+yTQCMIEz6THCAzABlByBkjxls0GwsUKHyN/jqqnXxIwwmZ5YZkjaq9cgL1uwyf/g3BRA2CwCCkqMgDCwoJU6LLEmNwwusFQ2SFJicLlUc/////OtnWOn3zqNySu1JPZc65Bk2OVAAABECAFjSfEiHFCZADQIAApmtLFQQXAImoEoAT/+2TE14KL0LFBjOEFQYafqDGmFXiq0+9P3T+dthMjJm+G2W922I4HTC05wnkzPRzcKotHtysZO2M0jmn1XAOgtwNgnhNy4Gcfj1C2xtsrTLd0o5Y23EIcJwEwxNuWyJN6jgbicYiEKWiy/cVkUfpvoP0MrmRdB4CICZhgAGDAQwNgHgDAnOelxbEBM5bW3HmJS6WS2/u5dq00b5Xx7lfzh21hlhVq8z/W8OWsZma5esTVupah2Yja/4itRiwXGio4OCl4l2xjOBoi7zawfa3qIrdn6xRJlaHExvYHpJ5mXnySLDQtOJ47w5Ptz0+iaur/+2TE5IIMjRM/jSTLQa+eZtG8pChTHTJiw02S317cFGkurbv+A4lKZWRIyAAIFAQlKoa8bitNJ4anZxweR6Xcn7sqtY1atcfjsyrrbCOjZspqVNq2UXjpweY4Rzg2VFE6cWTky6ST+iZIo1Gyjdm9zEyK2csTTNbK6na7KSdFjWZmROeJSwwroEiHWXkyiQAcA0y7HqTomiaPxMvMbRcBdIcxNashPsgtUK1zDQ5yqpG0O8ZUsdrPPuXPEAASBp08pnNVTv59bmx8b5kerudaCzlzMvuPdune6w1FHGSZGHRUACA+G2297VAAAmAJOaz/+2TE6YAP5RE3bT0PyiMi57Gns8Cuybp52DZS8OoBafA8AzbS5e1iQWuCVYhZ4GyEjjGBmEChkyRQPw4uaIwMjDI9YjBSZR3YA60QqT/Qs0x6zzQhDHB44MB0MRoP0MPyAoBuOKsa8ZsvxlZHZISsYEmtMkCJg/1chq2e627Ti/HLgnHC7lFcFekEq5wWxQSwouZCish4V2WIANNlzeypliE0BTnYE11lkDqqIasTgzJ12sAYuCIdY4DXCq3skB+9V6FKBkZMVfw41Mv5Ym8TWeMkdjzBj4isCNTMQwz3nVjSTVpRMBELLNnDYhOhfVz/+2TE0wBMcP9H7A2xCXEgZ3zxmhlQ8QmwWC8XqAOsuHzRGmDUROXGVwPplSbRGVQSmmFCUeHAkKCi00lwi00GECsmbHVrzKh2pWjAAAAYrWnewwaWAY802CdiERl9+pGoBkIcUBXe4RZ6VmZIivlroO1Y+FvnRUga62Q4ggtFXNpagVeRjdTEvCGXMbSZXVOSty5hg0581SWZbqsbc5OIu218LrUtOzCJrQAIESIZf+TQwsKnow6tAFJFZfD0/vc0EjmMacQok1+YEI1znxtemLvpCAQ8lOEgIOYmgcimDln2iVgtMjB28PQVRrUqoNv/+2TE4IBRMRM3jA3xCjCiaH2XpjHxyPEsGHoVeFf7SuSD4VVrRTb5KjghQlABIAAAERWyxl5ufBiMZzoxqPBSIYf98sCIhSg2VrSy64tQW50zSzRDJgVSprEECqFlskTEwyRPBImLmGCOzDLUNixq8Ymm5rT8ZX/f3u1ZqZA/RAVlFO9nmT7cWBZZBQwFocCjPGEX4lxzEvN85kEYjOhMRiUKgJ8ZWpxR6kXnwSAVdIrvO07KJCVFObYVJU1CplgLGkJYlITgJG3SWDSFVVR8ft/JU1F08IKKodf13y/wbqKChVypsWFLAgXI0WXvXr3/+2TEwwAL5O1R7AUQCXwfKXmDDfmCT0hS0crhdOnKsEGGUAdCxCUvPYHmunaDkBIRjIyJMS560K11dQrFq/M09acooRJivXHkyRpEFI5VkiVzsyaRJUbjy3l8OSkiDyoCaCwNHjobEWIlFY31hFlYA8xkhXZfNWa9igqkMbOqbHwv6c7qUJKXyFdNChJDPdR1csyvsSRrnTCut3LmJ0Ik+kzU312L25Sa5ZSECaIw6JdJlZEfZYdTaTjOrpNUKx3xyiRHU+kglF0YJVQT/n00rCiiVYkSSOzudtBWs1TzjyLJUxJLy8oog2olP6u3ueX/+2TE0QCL4Ms/x40wyX+V5vDzJhlpHISmDSSZpF801sRNnDtOojM+6pyplGaqmJd3iiSR2vTJOxI9Fnq2JLeZIkULynbG5qPeMYeFz4WM197esEuU1l9jstysBN5zuse2Ktw6vhAAQDi05WlaFmKi1NVrHc6HBxB93dAuiPPV18EEQRWhKhRfMi/fNNXyzYiXX2MqYIQpSE9z7JYVMyttsCMcifb/lh0hxCgLPPP5E577GspoZ+DlgiOFyEVbveYg64HKtbcYwwlb6hACAAEGS6jt6peXoou6T51EtyoOXS4OvQEMGY1KDE3U2JS0eH3/+2TE3wOMBL0cJ7DRQVEj4QTzDADVRqWXO0S+VNKi0TDMCMIxqR4YyqqWRpJ1DJt1TwakUZma/z46yrFybmRA8vyyVSMyacOFFYjErQtB5wxYuSOvuZShEglw2upAaSAJAjz3szohFQFkrBya1AmXaLnSUJuz09qnTe1R5LwkJJcQpoi2IbqREaueB13/WwDBgDGhlcqDaqhCOs4PNw9ddqa58hMMiFS4hjJtJr16pe7jkXUyxCoTA0JtSmYj7rT3HfGJYObMUNr3zUmHlh3aCBAAAACFywdCMzwbDmYrZDYwWKUsrVtG18kMeDExzDf/+2TE8gGPxTz+B4zCCZStX+BQjABBIGnRV1I8rC7WSphCTgJASaTPbBvnGmy9C6lTcrS4UiZTp55y+eVcKDtyy/yy+WUKk8OyHAzrx2bGAYNOXrDDdzy7K6XIKT6VgQCIhYrc7TWqQaiToiIDJGJkL2uwfKCwjG6NeGNHdjvSIcofVMuGftwERWvXDkvUuuaw0WFSIFRYwiFloBzB7m7j8S1V+RAkjIQWOwQ2abhzRaqTe0aQhkMBsIgU2FAUhk6lqZVT6mtyKnsl3KoAiAj0J+GMIAHrEpmiitJUgg1uREBu7CiFJmdXcoH+aZviIRL/+2TE7YAN7Xr/AwxyAcCwH8hhjBm0ktuiK+QfTcORbFiqVoak02KQQnaMpOQ9Y1BGJflshPIr7QX6uJTuTGQYs8teef04ea1qeopQvGMdqyk6wDJP8mp2s+b1e9oD9ZgBgAkeaoVdMOkEsOaLSwFeI1ZV0jOokhXqZcZ6MhGx3RKpa8imoCEYwZGpNmHXzFUnkJmwR8m0ZiEkzrKXY/CLaCVbv6dFaRSbHcr0lHpcuXDJDOTfEUsJDoQnDKiFj8OPmDG5izbBRx6MRSQICPIYKGCBwhyHIMZHUbY0rkOO2mLuapTMAu6bZGMR1obBLtn/+2TE6wANJXkBIYR+AbWqX+AxDwBT7eg3QzzVUZ43XUz7Hbo29IvCq7SxSBmSC8yUFXkYT9bI6HWjS2oxkWRrmW6cVZEUiwY2JC5rhUPiF/8t81KuyaFSVOHdOn2De47pdaHHQm5IylWI7ni3BCnKX5h8oStJx3YLGMkOsJCyOkWThpm325KCbBFCdhSiLRYzz7cgqApaEEAFavSoWmpaHeGVTpFLvKuO4lQ2lVjJG9YFMq1R4al8dr9+51uytu8+7P5OK+7/ufa7MirYUuqk36zgYC1jBmaqGPmSKudn84WjVTOUclrkaSrZyZZE7fn/+2TE7QBN1XL+oQRgCawt3+AxDZhTFEcNgkFbRaVdjUZ/NR7mwuYNy5o7Zk4lqPNRgpl2bDGrWNSal8qrGDKUON6w81VeLrOk3nYdJgpkiv9my4t/Tfhfwq6Nc3/8F4dQNymCgnlYKGBgwjysoIO6H8qGyxy9WDA0eWBQwYOjysFBAg6WKGCggTpysDRD//VZUP2ChQTp9DBQYJ5YoYGDR5WChgYNH+WO5f/s1j/qwao/xQQKpZWCggQORYWEeK+sWbi7MVFsWF2YrUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TE7YEOUc7+oYxtyaKln4AxD0FVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVMlbCiUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TE7QPOSWr4AZh1iZ2o2IAxjXhVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TElAPASAIADQAAIAcAQAGAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVX/+2TEkgPAAAGkAAAAIAAANIAAAARVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVU= + tools: + - functionDeclarations: + - name: structured_data + description: Result of transcribing an audio file + parameters: + type: object + properties: + transcription: + type: string + description: The transcription of the audio + required: + - transcription + tool_config: + function_calling_config: + mode: ANY + allowed_function_names: + - structured_data +step2RawChunks: + - candidates: + - content: + role: model + parts: + - functionCall: + name: structured_data + args: + transcription: Hello world + finishReason: STOP + index: 0 + usageMetadata: + promptTokenCount: 48 + candidatesTokenCount: 7 + totalTokenCount: 55 + promptTokensDetails: + - modality: TEXT + tokenCount: 23 + - modality: AUDIO + tokenCount: 25 + candidatesTokensDetails: + - modality: TEXT + tokenCount: 7 +step3KurtEvents: + - chunk: '{"transcription":"Hello world"}' + - finished: true + text: '{"transcription":"Hello world"}' + data: + transcription: Hello world + metadata: + totalInputTokens: 48 + totalOutputTokens: 7 diff --git a/packages/kurt-vertex-ai/src/KurtVertexAI.ts b/packages/kurt-vertex-ai/src/KurtVertexAI.ts index 0d5b2bc..b905f2b 100644 --- a/packages/kurt-vertex-ai/src/KurtVertexAI.ts +++ b/packages/kurt-vertex-ai/src/KurtVertexAI.ts @@ -1,9 +1,14 @@ import "./VertexAI.patch.generateContentStream" // monkey-patches VertexAI GenerativeModel.prototype.generateContentStream - import zodToJsonSchema from "zod-to-json-schema" import { type KurtAdapterV1, - type KurtStreamEvent, + KurtCapabilityError, + type KurtMessage, + type KurtResult, + KurtResultBlockedError, + KurtResultLimitError, + KurtResultValidateError, + type KurtSamplingOptions, type KurtSchema, type KurtSchemaInner, type KurtSchemaInnerMap, @@ -13,13 +18,7 @@ import { type KurtSchemaMaybe, type KurtSchemaResult, type KurtSchemaResultMaybe, - type KurtMessage, - type KurtSamplingOptions, - type KurtResult, - KurtResultValidateError, - KurtResultLimitError, - KurtResultBlockedError, - KurtCapabilityError, + type KurtStreamEvent, } from "@formula-monks/kurt" import type { VertexAI, @@ -27,7 +26,6 @@ import type { VertexAIMessage, VertexAIRequest, VertexAIResponseChunk, - VertexAIResponseChunkCandidate, VertexAIResponseFunctionCall, VertexAISchema, VertexAITool, @@ -171,7 +169,7 @@ function toVertexAIMessages(messages: KurtMessage[]): VertexAIMessage[] { const vertexAIMessages: VertexAIMessage[] = [] for (const message of messages) { - const { role, text, toolCall, imageData } = message + const { role, text, toolCall, imageData, inlineData } = message if (text) { vertexAIMessages.push({ role, parts: [{ text }] }) } else if (toolCall) { @@ -180,10 +178,10 @@ function toVertexAIMessages(messages: KurtMessage[]): VertexAIMessage[] { const functionResponse = { name, response: result } vertexAIMessages.push({ role, parts: [{ functionCall }] }) vertexAIMessages.push({ role, parts: [{ functionResponse }] }) - } else if (imageData) { - const { mimeType, base64Data } = imageData - const inlineData = { mimeType, data: base64Data } - vertexAIMessages.push({ role, parts: [{ inlineData }] }) + } else if (imageData || inlineData) { + const { mimeType, base64Data } = inlineData ?? imageData + const dataPart = { mimeType, data: base64Data } + vertexAIMessages.push({ role, parts: [{ inlineData: dataPart }] }) } else { throw new Error(`Invalid KurtMessage: ${JSON.stringify(message)}`) } diff --git a/packages/kurt-vertex-ai/src/VertexAI.patch.generateContentStream.ts b/packages/kurt-vertex-ai/src/VertexAI.patch.generateContentStream.ts index 1edb637..d2d115b 100644 --- a/packages/kurt-vertex-ai/src/VertexAI.patch.generateContentStream.ts +++ b/packages/kurt-vertex-ai/src/VertexAI.patch.generateContentStream.ts @@ -66,7 +66,6 @@ import { export async function generateContent( location: string, - project: string, publisherModelEndpoint: string, token: Promise, request: GenerateContentRequest | string, @@ -96,7 +95,6 @@ export async function generateContent( } const response: Response | undefined = await postRequest({ region: location, - project: project, resourcePath: publisherModelEndpoint, resourceMethod: constants.GENERATE_CONTENT_METHOD, token: await token, @@ -122,7 +120,6 @@ export async function generateContent( */ export async function generateContentStream( location: string, - project: string, publisherModelEndpoint: string, token: Promise, request: GenerateContentRequest | string, @@ -151,7 +148,6 @@ export async function generateContentStream( } const response = await postRequest({ region: location, - project: project, resourcePath: publisherModelEndpoint, resourceMethod: constants.STREAMING_GENERATE_CONTENT_METHOD, token: await token, @@ -183,7 +179,6 @@ async function generateContentStreamPATCHED( const _this = this as any return generateContentStream( _this.location, - _this.project, _this.publisherModelEndpoint, _this.fetchToken(), request, diff --git a/packages/kurt/src/Kurt.ts b/packages/kurt/src/Kurt.ts index 54570e9..f92445b 100644 --- a/packages/kurt/src/Kurt.ts +++ b/packages/kurt/src/Kurt.ts @@ -98,7 +98,7 @@ export class Kurt { * - autonomous, open-ended decision-making or action-taking * * The `data` field of the result will be `undefined` if Kurt decides to - * generate natural language. Otherwise it will contain a tool call. + * generate natural language. Otherwise, it will contain a tool call. * * Your application can decide if and how it should fulfill the tool call. * @@ -198,25 +198,23 @@ export type KurtMessage = { text: string /** + * @deprecated Use `inlineData` instead. * When present, this is an image data message, with a base64-encoded image. * This is often used with "multi-modal" LLMs that support image mode input. * * Not all LLM providers or underlying models support this kind of message. - * Check your LLM provider's documentation for confirmaton. + * Check your LLM provider's documentation for confirmation. */ - imageData: { - /** - * The IANA standard MIME type of the inline image data. - * - * Not all MIME types are supported by all LLM providers. - * "image/png" and "image/jpeg" are the most commonly supported. - * Check your LLM provider's documentation for the right list. - */ - mimeType: string + imageData: KurtInlineData - /** Base64-encoded image data, as a string. */ - base64Data: string - } + /** + * When present, this is a base64-encoded data message (i.e. image, audio). + * This is often used with "multi-modal" LLMs that support image/audio mode input. + * + * Not all LLM providers or underlying models support this kind of message. + * Check your LLM provider's documentation for confirmation. + */ + inlineData: KurtInlineData /** * When present, this is a tool call message, with structured data input @@ -251,6 +249,21 @@ export type KurtMessage = { } }> +export interface KurtInlineData { + /** + * The IANA standard MIME type of the inline data. + * + * Not all MIME types are supported by all LLM providers. + * OpenAI's GPT, for example, supports only images. + * Gemini supports both images and audio. + * Check your LLM provider's documentation for the right list. + */ + mimeType: string + + /** Base64-encoded data, as a string. */ + base64Data: string +} + export interface KurtCreateOptions { /** * The default system prompt to use, for any generation method call which @@ -294,7 +307,7 @@ export const KurtSamplingOptionsDefault = { /** * Maximum number of output tokens to sample from the model. * - * This is mean to be a cost control measure, to protect against scenarios + * This is meant to be a cost control measure, to protect against scenarios * where the model might get "stuck" and generate excessive output. * * When the model hits the output limit, whatever it has generated will diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02bc197..aa95136 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,11 +48,11 @@ importers: specifier: workspace:* version: link:../../packages/kurt-vertex-ai '@google-cloud/vertexai': - specifier: 1.1.0 - version: 1.1.0 + specifier: 1.9.3 + version: 1.9.3 openai: - specifier: ^4.76.0 - version: 4.76.0(zod@3.23.8) + specifier: 4.85.1 + version: 4.85.1(zod@3.23.8) zod: specifier: ^3.23.8 version: 3.23.8 @@ -156,8 +156,8 @@ importers: specifier: workspace:^ version: link:../kurt openai: - specifier: 4.76.0 - version: 4.76.0(zod@3.23.8) + specifier: 4.85.1 + version: 4.85.1(zod@3.23.8) zod: specifier: ^3.23.8 version: 3.23.8 @@ -199,8 +199,8 @@ importers: specifier: workspace:^ version: link:../kurt '@google-cloud/vertexai': - specifier: 1.1.0 - version: 1.1.0 + specifier: 1.9.3 + version: 1.9.3 zod: specifier: ^3.23.8 version: 3.23.8 @@ -673,8 +673,8 @@ packages: cpu: [x64] os: [win32] - '@google-cloud/vertexai@1.1.0': - resolution: {integrity: sha512-hfwfdlVpJ+kM6o2b5UFfPnweBcz8tgHAFRswnqUKYqLJsvKU0DDD0Z2/YKoHyAUoPJAv20qg6KlC3msNeUKUiw==} + '@google-cloud/vertexai@1.9.3': + resolution: {integrity: sha512-35o5tIEMLW3JeFJOaaMNR2e5sq+6rpnhrF97PuAxeOm0GlqVTESKhkGj7a5B5mmJSSSU3hUfIhcQCRRsw4Ipzg==} engines: {node: '>=18.0.0'} '@istanbuljs/load-nyc-config@1.1.0': @@ -2519,12 +2519,15 @@ packages: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} engines: {node: '>=12'} - openai@4.76.0: - resolution: {integrity: sha512-QBGIetjX1C9xDp5XGa/3mPnfKI9BgAe2xHQX6PmO98wuW9qQaurBaumcYptQWc9LHZZq7cH/Y1Rjnsr6uUDdVw==} + openai@4.85.1: + resolution: {integrity: sha512-jkX2fntHljUvSH3MkWh4jShl10oNkb+SsCj4auKlbu2oF4KWAnmHLNR5EpnUHK1ZNW05Rp0fjbJzYwQzMsH8ZA==} hasBin: true peerDependencies: + ws: ^8.18.0 zod: ^3.23.8 peerDependenciesMeta: + ws: + optional: true zod: optional: true @@ -3848,7 +3851,7 @@ snapshots: '@esbuild/win32-x64@0.21.5': optional: true - '@google-cloud/vertexai@1.1.0': + '@google-cloud/vertexai@1.9.3': dependencies: google-auth-library: 9.11.0 transitivePeerDependencies: @@ -6063,7 +6066,7 @@ snapshots: dependencies: mimic-fn: 4.0.0 - openai@4.76.0(zod@3.23.8): + openai@4.85.1(zod@3.23.8): dependencies: '@types/node': 18.19.32 '@types/node-fetch': 2.6.11