Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/pr-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24.3"
go-version: "1.25.5"

- name: Set up Python
uses: actions/setup-python@v5
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/release-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24.3"
go-version: "1.25.5"
- name: Configure Git
run: |
git config user.name "GitHub Actions Bot"
Expand Down Expand Up @@ -139,7 +139,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24.3"
go-version: "1.25.5"

- name: Configure Git
run: |
Expand Down Expand Up @@ -215,7 +215,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24.3"
go-version: "1.25.5"

- name: Configure Git
run: |
Expand Down Expand Up @@ -294,7 +294,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24.3"
go-version: "1.25.5"

- name: Set up Node.js
uses: actions/setup-node@v4
Expand Down
22 changes: 11 additions & 11 deletions .github/workflows/snyk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ name: Snyk checks

on:
push:
branches: [main, master, '**/*']
branches: [main, master, "**/*"]
pull_request:
branches: ['**/*']
branches: ["**/*"]
workflow_dispatch:

permissions:
Expand Down Expand Up @@ -44,17 +44,17 @@ jobs:
- name: Setup Node (for UI)
uses: actions/setup-node@v4
with:
node-version: '20'
node-version: "20"

- name: Setup Python (for tests tooling)
uses: actions/setup-python@v5
with:
python-version: '3.11'
python-version: "3.11"

- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.24.3'
go-version: "1.25.5"

- name: Install Snyk CLI
uses: snyk/actions/setup@master
Expand Down Expand Up @@ -82,22 +82,22 @@ jobs:
- name: Setup Node (for UI)
uses: actions/setup-node@v4
with:
node-version: '20'
node-version: "20"

- name: Setup Python (for tests tooling)
uses: actions/setup-python@v5
with:
python-version: '3.11'
python-version: "3.11"

- name: Setup Python (for tests tooling)
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
python-version: "3.11"
cache: "pip"
cache-dependency-path: |
tests/integrations/requirements.txt
tests/governance/requirements.txt

- name: Install Python dependencies (tests tooling)
run: |
python -m pip install --disable-pip-version-check \
Expand All @@ -107,7 +107,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.24.3'
go-version: "1.25.5"

- name: Build
run: make build
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ _build-with-docker: # Internal target for Docker-based cross-compilation
-e GOOS=$(TARGET_OS) \
-e GOARCH=$(TARGET_ARCH) \
$(if $(LOCAL),,-e GOWORK=off) \
golang:1.24.3-alpine3.22 \
golang:1.25.5-alpine3.22 \
sh -c "apk add --no-cache gcc musl-dev && \
go build \
-ldflags='-w -s -X main.Version=v$(VERSION)' \
Expand All @@ -230,7 +230,7 @@ _build-with-docker: # Internal target for Docker-based cross-compilation
-e GOOS=$(TARGET_OS) \
-e GOARCH=$(TARGET_ARCH) \
$(if $(LOCAL),,-e GOWORK=off) \
golang:1.24.3-alpine3.22 \
golang:1.25.5-alpine3.22 \
sh -c "apk add --no-cache gcc musl-dev && \
go build \
-ldflags='-w -s -extldflags "-static" -X main.Version=v$(VERSION)' \
Expand Down
72 changes: 48 additions & 24 deletions core/internal/testutil/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type TestScenarios struct {
FileDelete bool // File API delete functionality
FileContent bool // File API content download functionality
FileBatchInput bool // Whether batch create supports file-based input (InputFileID)
ChatAudio bool // Chat completion with audio input/output functionality
}

// ComprehensiveTestConfig extends TestConfig with additional scenarios
Expand All @@ -66,13 +67,16 @@ type ComprehensiveTestConfig struct {
EmbeddingModel string
TranscriptionModel string
SpeechSynthesisModel string
ChatAudioModel string
Scenarios TestScenarios
Fallbacks []schemas.Fallback // for chat, responses, image and reasoning tests
TextCompletionFallbacks []schemas.Fallback // for text completion tests
TranscriptionFallbacks []schemas.Fallback // for transcription tests
SpeechSynthesisFallbacks []schemas.Fallback // for speech synthesis tests
EmbeddingFallbacks []schemas.Fallback // for embedding tests
SkipReason string // Reason to skip certain tests
ExternalTTSProvider schemas.ModelProvider // External TTS provider to use for testing
ExternalTTSModel string // External TTS model to use for testing
BatchExtraParams map[string]interface{} // Extra params for batch operations (e.g., role_arn, output_s3_uri for Bedrock)
FileExtraParams map[string]interface{} // Extra params for file operations (e.g., s3_bucket for Bedrock)
}
Expand Down Expand Up @@ -161,25 +165,25 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx *context.Context
},
},
},
{
Models: []string{},
Weight: 1.0,
BedrockKeyConfig: &schemas.BedrockKeyConfig{
AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
SessionToken: bifrost.Ptr(os.Getenv("AWS_SESSION_TOKEN")),
Region: bifrost.Ptr(getEnvWithDefault("AWS_REGION", "us-east-1")),
ARN: bifrost.Ptr(os.Getenv("AWS_BEDROCK_ARN")),
Deployments: map[string]string{
"claude-3.5-sonnet": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
"claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0",
"claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0",
"claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0",
{
Models: []string{},
Weight: 1.0,
BedrockKeyConfig: &schemas.BedrockKeyConfig{
AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
SessionToken: bifrost.Ptr(os.Getenv("AWS_SESSION_TOKEN")),
Region: bifrost.Ptr(getEnvWithDefault("AWS_REGION", "us-east-1")),
ARN: bifrost.Ptr(os.Getenv("AWS_BEDROCK_ARN")),
Deployments: map[string]string{
"claude-3.5-sonnet": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"claude-3.7-sonnet": "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
"claude-4-sonnet": "global.anthropic.claude-sonnet-4-20250514-v1:0",
"claude-4.5-sonnet": "global.anthropic.claude-sonnet-4-5-20250929-v1:0",
"claude-4.5-haiku": "global.anthropic.claude-haiku-4-5-20251001-v1:0",
},
},
UseForBatchAPI: bifrost.Ptr(true),
},
UseForBatchAPI: bifrost.Ptr(true),
},
{
Models: []string{"cohere.embed-v4:0"},
Weight: 1.0,
Expand Down Expand Up @@ -218,6 +222,20 @@ func (account *ComprehensiveTestAccount) GetKeysForProvider(ctx *context.Context
},
UseForBatchAPI: bifrost.Ptr(true),
},
{
Value: os.Getenv("AZURE_API_KEY"),
Models: []string{},
Weight: 1.0,
AzureKeyConfig: &schemas.AzureKeyConfig{
Endpoint: os.Getenv("AZURE_ENDPOINT"),
APIVersion: bifrost.Ptr("2025-01-01-preview"),
Deployments: map[string]string{
"whisper": "whisper",
"gpt-4o-mini-tts": "gpt-4o-mini-tts",
"gpt-4o-mini-audio-preview": "gpt-4o-mini-audio-preview",
},
},
},
}, nil
case schemas.Vertex:
return []schemas.Key{
Expand Down Expand Up @@ -587,6 +605,7 @@ var AllProviderConfigs = []ComprehensiveTestConfig{
PromptCachingModel: "gpt-4.1",
TranscriptionModel: "whisper-1",
SpeechSynthesisModel: "tts-1",
ChatAudioModel: "gpt-4o-mini-audio-preview",
Scenarios: TestScenarios{
TextCompletion: false, // Not supported
TextCompletionStream: false, // Not supported
Expand Down Expand Up @@ -618,6 +637,7 @@ var AllProviderConfigs = []ComprehensiveTestConfig{
FileRetrieve: true, // OpenAI supports file API
FileDelete: true, // OpenAI supports file API
FileContent: true, // OpenAI supports file API
ChatAudio: true, // OpenAI supports chat audio
},
Fallbacks: []schemas.Fallback{
{Provider: schemas.Anthropic, Model: "claude-3-7-sonnet-20250219"},
Expand Down Expand Up @@ -725,9 +745,12 @@ var AllProviderConfigs = []ComprehensiveTestConfig{
},
},
{
Provider: schemas.Azure,
ChatModel: "gpt-4o",
TextModel: "", // Azure doesn't support text completion in newer models
Provider: schemas.Azure,
ChatModel: "gpt-4o",
TextModel: "", // Azure doesn't support text completion in newer models
ChatAudioModel: "gpt-4o-mini-audio-preview",
TranscriptionModel: "whisper-1",
SpeechSynthesisModel: "gpt-4o-mini-tts",
Scenarios: TestScenarios{
TextCompletion: false, // Not supported
SimpleChat: true,
Expand All @@ -741,10 +764,10 @@ var AllProviderConfigs = []ComprehensiveTestConfig{
ImageBase64: true,
MultipleImages: true,
CompleteEnd2End: true,
SpeechSynthesis: false, // Not supported yet
SpeechSynthesisStream: false, // Not supported yet
Transcription: false, // Not supported yet
TranscriptionStream: false, // Not supported yet
SpeechSynthesis: true, // Supported via gpt-4o-mini-tts
SpeechSynthesisStream: true, // Supported via gpt-4o-mini-tts
Transcription: true, // Supported via whisper-1
TranscriptionStream: false, // Not properly supported yet by Azure
Embedding: true,
ListModels: true,
BatchCreate: true, // Azure supports batch API
Expand All @@ -757,6 +780,7 @@ var AllProviderConfigs = []ComprehensiveTestConfig{
FileRetrieve: true, // Azure supports file API
FileDelete: true, // Azure supports file API
FileContent: true, // Azure supports file API
ChatAudio: true, // Azure supports chat audio
},
Fallbacks: []schemas.Fallback{
{Provider: schemas.OpenAI, Model: "gpt-4o-mini"},
Expand Down
Loading