fix: missing image url and audio struct in chat completion gemini

core/changelog.md

@@ -0,0 +1,2 @@
+- fix: image url and input audio handling in gemini chat converters
+- fix: support both responseJsonSchema and responseSchema for JSON response formatting in gemini

core/internal/testutil/account.go

@@ -250,7 +250,7 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx *context.Context
 		return []schemas.Key{
 			{
 				Value:  os.Getenv("VERTEX_API_KEY"),
-				Models: []string{},
+				Models: []string{"text-multilingual-embedding-002", "google/gemini-2.0-flash-001"},
 				Weight: 1.0,
 				VertexKeyConfig: &schemas.VertexKeyConfig{
 					ProjectID:       os.Getenv("VERTEX_PROJECT_ID"),
@@ -259,6 +259,17 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx *context.Context
 				},
 				UseForBatchAPI: bifrost.Ptr(true),
 			},
+			{
+				Value:  os.Getenv("VERTEX_API_KEY"),
+				Models: []string{"claude-sonnet-4-5", "claude-4.5-haiku", "claude-opus-4-5"},
+				Weight: 1.0,
+				VertexKeyConfig: &schemas.VertexKeyConfig{
+					ProjectID:       os.Getenv("VERTEX_PROJECT_ID"),
+					Region:          getEnvWithDefault("VERTEX_REGION_ANTHROPIC", "us-east5"),
+					AuthCredentials: os.Getenv("VERTEX_CREDENTIALS"),
+				},
+				UseForBatchAPI: bifrost.Ptr(true),
+			},
 		}, nil
 	case schemas.Mistral:
 		return []schemas.Key{

core/internal/testutil/image_base64.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 	"testing"
 
-
 	bifrost "github.com/maximhq/bifrost/core"
 	"github.com/maximhq/bifrost/core/schemas"
 )
@@ -72,7 +71,7 @@ func RunImageBase64Test(t *testing.T, client *bifrost.Bifrost, ctx context.Conte
 				Model:    testConfig.VisionModel,
 				Input:    chatMessages,
 				Params: &schemas.ChatParameters{
-					MaxCompletionTokens: bifrost.Ptr(200),
+					MaxCompletionTokens: bifrost.Ptr(500),
 				},
 				Fallbacks: testConfig.Fallbacks,
 			}
@@ -85,7 +84,7 @@ func RunImageBase64Test(t *testing.T, client *bifrost.Bifrost, ctx context.Conte
 				Model:    testConfig.VisionModel,
 				Input:    responsesMessages,
 				Params: &schemas.ResponsesParameters{
-					MaxOutputTokens: bifrost.Ptr(200),
+					MaxOutputTokens: bifrost.Ptr(500),
 				},
 				Fallbacks: testConfig.Fallbacks,
 			}
@@ -146,12 +145,13 @@ func validateBase64ImageContent(t *testing.T, content string, apiName string) {
 		strings.Contains(lowerContent, "cat") || strings.Contains(lowerContent, "feline")
 
 	if len(content) < 10 {
-		t.Logf("⚠️ %s response seems quite short for image description: %s", apiName, content)
-	} else if foundAnimal {
-		t.Logf("✅ %s vision model successfully identified animal in base64 image", apiName)
-	} else {
-		t.Logf("✅ %s vision model processed base64 image but may not have clearly identified the animal", apiName)
+		t.Fatalf("❌ %s response too short for image description: %s", apiName, content)
+	}
+
+	if !foundAnimal {
+		t.Fatalf("❌ %s vision model failed to identify any animal in base64 image: %s", apiName, content)
 	}
 
+	t.Logf("✅ %s vision model successfully identified animal in base64 image", apiName)
 	t.Logf("✅ %s lion base64 image processing completed: %s", apiName, content)
 }

core/providers/anthropic/responses.go

@@ -3462,7 +3462,10 @@ func (block AnthropicContentBlock) toBifrostResponsesDocumentBlock() schemas.Res
 			if block.Source.MediaType != nil {
 				mediaType = *block.Source.MediaType
 			}
-			dataURL := "data:" + mediaType + ";base64," + *block.Source.Data
+			dataURL := *block.Source.Data
+			if !strings.HasPrefix(dataURL, "data:") {
+				dataURL = "data:" + mediaType + ";base64," + *block.Source.Data
+			}
 			resultBlock.ResponsesInputMessageContentBlockFile.FileData = &dataURL
 		}
 	case "text":

core/providers/bedrock/responses.go

@@ -2728,9 +2728,9 @@ func convertSingleBedrockMessageToBifrostMessages(ctx *context.Context, msg *Bed
 				fileBlock.ResponsesInputMessageContentBlockFile.Filename = &block.Document.Name
 			}
 
+			fileType := "application/pdf"
 			// Set file type based on format
 			if block.Document.Format != "" {
-				var fileType string
 				switch block.Document.Format {
 				case "pdf":
 					fileType = "application/pdf"
@@ -2749,7 +2749,11 @@ func convertSingleBedrockMessageToBifrostMessages(ctx *context.Context, msg *Bed
 					fileBlock.ResponsesInputMessageContentBlockFile.FileData = block.Document.Source.Text
 				} else if block.Document.Source.Bytes != nil {
 					// Base64 encoded bytes (PDF)
-					fileBlock.ResponsesInputMessageContentBlockFile.FileData = block.Document.Source.Bytes
+					fileDataURL := *block.Document.Source.Bytes
+					if !strings.HasPrefix(fileDataURL, "data:") {
+						fileDataURL = fmt.Sprintf("data:%s;base64,%s", fileType, fileDataURL)
+					}
+					fileBlock.ResponsesInputMessageContentBlockFile.FileData = &fileDataURL
 				}
 			}
 

core/providers/gemini/chat.go

@@ -57,7 +57,11 @@ func ToGeminiChatCompletionRequest(bifrostReq *schemas.BifrostChatRequest) *Gemi
 	}
 
 	// Convert chat completion messages to Gemini format
-	geminiReq.Contents = convertBifrostMessagesToGemini(bifrostReq.Input)
+	contents, systemInstruction := convertBifrostMessagesToGemini(bifrostReq.Input)
+	if systemInstruction != nil {
+		geminiReq.SystemInstruction = systemInstruction
+	}
+	geminiReq.Contents = contents
 
 	return geminiReq
 }

core/providers/gemini/gemini_test.go

@@ -31,7 +31,7 @@ func TestGemini(t *testing.T) {
 		Fallbacks: []schemas.Fallback{
 			{Provider: schemas.Gemini, Model: "gemini-2.5-flash"},
 		},
-		VisionModel:          "gemini-2.0-flash",
+		VisionModel:          "gemini-2.5-flash",
 		EmbeddingModel:       "text-embedding-004",
 		TranscriptionModel:   "gemini-2.5-flash",
 		SpeechSynthesisModel: "gemini-2.5-flash-preview-tts",

core/providers/gemini/responses.go

@@ -726,7 +726,7 @@ func (response *GenerateContentResponse) ToBifrostResponsesStream(sequenceNumber
 		// Check for finish reason (indicates end of generation)
 		// Only close if we've actually started emitting content (text, tool calls, etc.)
 		// This prevents emitting response.completed for empty chunks with just finishReason
-		if candidate.FinishReason != "" && (state.HasStartedText || state.HasStartedToolCall) {
+		if candidate.FinishReason != "" && len(state.ItemIDs) > 0 {
 			// Close any open items
 			closeResponses := closeGeminiOpenItems(state, response.UsageMetadata, sequenceNumber+len(responses))
 			responses = append(responses, closeResponses...)
@@ -1578,18 +1578,27 @@ func convertGeminiInlineDataToContentBlock(blob *Blob) *schemas.ResponsesMessage
 		}
 	}
 
-	// Handle other files
-	encodedData := base64.StdEncoding.EncodeToString(blob.Data)
+	// Handle other files - format as data URL
+	mimeTypeForFile := mimeType
+	if mimeTypeForFile == "" {
+		mimeTypeForFile = "application/pdf"
+	}
+
+	filename := blob.DisplayName
+	if filename == "" {
+		filename = "unnamed_file"
+	}
+
+	fileDataURL := base64.StdEncoding.EncodeToString(blob.Data)
+	if !strings.HasPrefix(fileDataURL, "data:") {
+		fileDataURL = fmt.Sprintf("data:%s;base64,%s", mimeTypeForFile, fileDataURL)
+	}
 	return &schemas.ResponsesMessageContentBlock{
 		Type: schemas.ResponsesInputMessageContentBlockTypeFile,
 		ResponsesInputMessageContentBlockFile: &schemas.ResponsesInputMessageContentBlockFile{
-			FileData: &encodedData,
-			FileType: func() *string {
-				if blob.MIMEType != "" {
-					return &blob.MIMEType
-				}
-				return nil
-			}(),
+			FileData: &fileDataURL,
+			FileType: &mimeTypeForFile,
+			Filename: &filename,
 		},
 	}
 }

core/providers/gemini/utils.go

@@ -772,10 +772,36 @@ func addSpeechConfigToGenerationConfig(config *GenerationConfig, voiceConfig *sc
 }
 
 // convertBifrostMessagesToGemini converts Bifrost messages to Gemini format
-func convertBifrostMessagesToGemini(messages []schemas.ChatMessage) []Content {
+func convertBifrostMessagesToGemini(messages []schemas.ChatMessage) ([]Content, *Content) {
 	var contents []Content
+	var systemInstruction *Content
 
 	for _, message := range messages {
+		// Handle system messages separately - Gemini requires them in SystemInstruction field
+		if message.Role == schemas.ChatMessageRoleSystem {
+			if systemInstruction == nil {
+				systemInstruction = &Content{}
+			}
+
+			// Extract system message content
+			if message.Content != nil {
+				if message.Content.ContentStr != nil && *message.Content.ContentStr != "" {
+					systemInstruction.Parts = append(systemInstruction.Parts, &Part{
+						Text: *message.Content.ContentStr,
+					})
+				} else if message.Content.ContentBlocks != nil {
+					for _, block := range message.Content.ContentBlocks {
+						if block.Text != nil && *block.Text != "" {
+							systemInstruction.Parts = append(systemInstruction.Parts, &Part{
+								Text: *block.Text,
+							})
+						}
+					}
+				}
+			}
+			continue
+		}
+
 		var parts []*Part
 
 		// Handle content
@@ -826,8 +852,74 @@ func convertBifrostMessagesToGemini(messages []schemas.ChatMessage) []Content {
 								})
 							}
 						}
+					} else if block.ImageURLStruct != nil {
+						// Handle image blocks
+						imageURL := block.ImageURLStruct.URL
+
+						// Sanitize and parse the image URL
+						sanitizedURL, err := schemas.SanitizeImageURL(imageURL)
+						if err != nil {
+							// Skip this block if URL is invalid
+							continue
+						}
+
+						urlInfo := schemas.ExtractURLTypeInfo(sanitizedURL)
+
+						// Determine MIME type
+						mimeType := "image/jpeg" // default
+						if urlInfo.MediaType != nil {
+							mimeType = *urlInfo.MediaType
+						}
+
+						if urlInfo.Type == schemas.ImageContentTypeBase64 {
+							// Data URL - convert to InlineData (Blob)
+							if urlInfo.DataURLWithoutPrefix != nil {
+								decodedData, err := base64.StdEncoding.DecodeString(*urlInfo.DataURLWithoutPrefix)
+								if err == nil && len(decodedData) > 0 {
+									parts = append(parts, &Part{
+										InlineData: &Blob{
+											MIMEType: mimeType,
+											Data:     decodedData,
+										},
+									})
+								}
+							}
+						} else {
+							// Regular URL - use FileData
+							parts = append(parts, &Part{
+								FileData: &FileData{
+									MIMEType: mimeType,
+									FileURI:  sanitizedURL,
+								},
+							})
+						}
+					} else if block.InputAudio != nil {
+						// Decode the audio data (already base64 encoded in the schema)
+						decodedData, err := base64.StdEncoding.DecodeString(block.InputAudio.Data)
+						if err != nil || len(decodedData) == 0 {
+							continue
+						}
+
+						// Determine MIME type
+						mimeType := "audio/mpeg" // default
+						if block.InputAudio.Format != nil {
+							format := strings.ToLower(strings.TrimSpace(*block.InputAudio.Format))
+							if format != "" {
+								if strings.HasPrefix(format, "audio/") {
+									mimeType = format
+								} else {
+									mimeType = "audio/" + format
+								}
+							}
+						}
+
+						parts = append(parts, &Part{
+							InlineData: &Blob{
+								MIMEType: mimeType,
+								Data:     decodedData,
+							},
+						})
 					}
-					// Handle other content block types as needed
 				}
 			}
 		}
@@ -945,7 +1037,7 @@ func convertBifrostMessagesToGemini(messages []schemas.ChatMessage) []Content {
 		}
 	}
 
-	return contents
+	return contents, systemInstruction
 }
 
 // normalizeSchemaTypes recursively normalizes type values from uppercase to lowercase

core/providers/vertex/vertex_test.go

@@ -25,10 +25,10 @@ func TestVertex(t *testing.T) {
 	testConfig := testutil.ComprehensiveTestConfig{
 		Provider:       schemas.Vertex,
 		ChatModel:      "google/gemini-2.0-flash-001",
-		VisionModel:    "gemini-2.0-flash-001",
+		VisionModel:    "claude-sonnet-4-5",
 		TextModel:      "", // Vertex doesn't support text completion in newer models
 		EmbeddingModel: "text-multilingual-embedding-002",
-		ReasoningModel: "claude-4.5-haiku",
+		ReasoningModel: "claude-opus-4-5",
 		Scenarios: testutil.TestScenarios{
 			TextCompletion:        false, // Not supported
 			SimpleChat:            true,
@@ -39,13 +39,13 @@ func TestVertex(t *testing.T) {
 			MultipleToolCalls:     true,
 			End2EndToolCalling:    true,
 			AutomaticFunctionCall: true,
-			ImageURL:              true,
+			ImageURL:              false,
 			ImageBase64:           true,
-			MultipleImages:        true,
+			MultipleImages:        false,
 			CompleteEnd2End:       true,
 			FileBase64:            true,
 			Embedding:             true,
-			Reasoning:             false, // Not supported right now because we are not using native gemini converters
+			Reasoning:             true,
 			ListModels:            false,
 			CountTokens:           true,
 			StructuredOutputs:     true, // Structured outputs with nullable enum support

core/version

@@ -1 +1 @@
-1.2.47
+1.2.48

framework/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48

framework/version

@@ -1 +1 @@
-1.1.59
+1.1.60

plugins/governance/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/governance/version

@@ -1 +1 @@
-1.3.60
+1.3.61

plugins/jsonparser/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/jsonparser/version

@@ -1 +1 @@
-1.3.60
+1.3.61

plugins/logging/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/logging/version

@@ -1 +1 @@
-1.3.60
+1.3.61

plugins/maxim/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/maxim/version

@@ -1 +1 @@
-1.4.61
+1.4.62

plugins/mocker/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/mocker/version

@@ -1 +1 @@
-1.3.58
+1.3.59

plugins/otel/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60

plugins/otel/version

@@ -1 +1 @@
-1.0.59
+1.0.60

plugins/semanticcache/changelog.md

@@ -0,0 +1 @@
+- chore: upgrades core to v1.2.48 and framework to 1.1.60