feat: saturn query pipeline for rag optimisation

go.mod

@@ -3,6 +3,8 @@ module github.com/tuannvm/slack-mcp-client
 go 1.24.4
 
 require (
+	github.com/aws/aws-sdk-go-v2/config v1.29.4
+	github.com/aws/aws-sdk-go-v2/service/s3vectors v1.4.10
 	github.com/joho/godotenv v1.5.1
 	github.com/mark3labs/mcp-go v0.42.0
 	github.com/openai/openai-go v1.8.2
@@ -24,6 +26,18 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.39.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.57 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
+	github.com/aws/smithy-go v1.23.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -34,6 +34,34 @@ github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZ
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg=
+github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
+github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg=
+github.com/aws/aws-sdk-go-v2/config v1.29.4/go.mod h1:j2/AF7j/qxVmsNIChw1tWfsVKOayJoGRDjg1Tgq7NPk=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9TIj3tS4MuP5b+TcQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
+github.com/aws/aws-sdk-go-v2/service/s3vectors v1.4.10 h1:hgJrhznAL6SjFZAqNIexiE9L7Zjc5PMGmwPWNtTE3zc=
+github.com/aws/aws-sdk-go-v2/service/s3vectors v1.4.10/go.mod h1:gJNoydxeaa5Av62mqcKTcA/9oFJnnZRseWfDmPKfGv8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13/go.mod h1:tvqlFoja8/s0o+UruA1Nrezo/df0PzdunMDDurUfg6U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 h1:fqg6c1KVrc3SYWma/egWue5rKI4G2+M4wMQN2JosNAA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.12/go.mod h1:7Yn+p66q/jt38qMoVfNvjbm3D89mGBnkwDcijgtih8w=
+github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
+github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=

helm-chart/slack-mcp-client/templates/deployment.yaml

@@ -24,9 +24,10 @@ spec:
       {{- end }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      {{- if and .Values.serviceAccount.create .Values.serviceAccount.clusterRoleName }}
-      serviceAccountName: {{ include "slack-mcp-client.fullname" . }}
+      {{- if .Values.serviceAccount.create }}
+      serviceAccountName: {{ .Values.serviceAccount.name | default (include "slack-mcp-client.fullname" .) }}
       {{- end }}
+
       {{- if .Values.initContainers }}
       initContainers:
         {{- range .Values.initContainers }}

helm-chart/slack-mcp-client/templates/service-account.yaml

@@ -1,8 +1,12 @@
-{{- if and .Values.serviceAccount.create .Values.serviceAccount.clusterRoleName }}
+{{- if .Values.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ include "slack-mcp-client.fullname" . }}
+  name: {{ .Values.serviceAccount.name | default (include "slack-mcp-client.fullname" .) }}
   labels:
     {{- include "slack-mcp-client.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

internal/config/config.go

@@ -22,17 +22,19 @@ const (
 
 // Config represents the main application configuration
 type Config struct {
-	Version        string                     `json:"version"`
-	Slack          SlackConfig                `json:"slack"`
-	LLM            LLMConfig                  `json:"llm"`
-	MCPServers     map[string]MCPServerConfig `json:"mcpServers"`
-	RAG            RAGConfig                  `json:"rag,omitempty"`
-	Monitoring     MonitoringConfig           `json:"monitoring,omitempty"`
-	Timeouts       TimeoutConfig              `json:"timeouts,omitempty"`
-	Retry          RetryConfig                `json:"retry,omitempty"`
-	Reload         ReloadConfig               `json:"reload,omitempty"`
-	Observability  ObservabilityConfig        `json:"observability,omitempty"`
-	UseStdIOClient bool                       `json:"useStdIOClient,omitempty"` // Use terminal client instead of a real slack bot, for local development
+	Version                    string                     `json:"version"`
+	Slack                      SlackConfig                `json:"slack"`
+	LLM                        LLMConfig                  `json:"llm"`
+	MCPServers                 map[string]MCPServerConfig `json:"mcpServers"`
+	QueryEnhancementProvider   string                     `json:"queryEnhancementProvider,omitempty"`   // Optional: LLM provider for query enhancement (applies to all queries)
+	QueryEnhancementPromptFile string                     `json:"queryEnhancementPromptFile,omitempty"` // Optional: Path to custom query enhancement prompt file
+	RAG                        RAGConfig                  `json:"rag,omitempty"`
+	Monitoring                 MonitoringConfig           `json:"monitoring,omitempty"`
+	Timeouts                   TimeoutConfig              `json:"timeouts,omitempty"`
+	Retry                      RetryConfig                `json:"retry,omitempty"`
+	Reload                     ReloadConfig               `json:"reload,omitempty"`
+	Observability              ObservabilityConfig        `json:"observability,omitempty"`
+	UseStdIOClient             bool                       `json:"useStdIOClient,omitempty"` // Use terminal client instead of a real slack bot, for local development
 }
 
 // SlackConfig contains Slack-specific configuration
@@ -109,26 +111,37 @@ type MCPToolsConfig struct {
 
 // RAGConfig contains RAG system configuration
 type RAGConfig struct {
-	Enabled   bool                         `json:"enabled,omitempty"`
-	Provider  string                       `json:"provider,omitempty"`
-	ChunkSize int                          `json:"chunkSize,omitempty"`
-	Providers map[string]RAGProviderConfig `json:"providers,omitempty"`
+	Enabled            bool                                  `json:"enabled,omitempty"`
+	Provider           string                                `json:"provider,omitempty"`
+	ChunkSize          int                                   `json:"chunkSize,omitempty"`
+	EmbeddingProvider  string                                `json:"embeddingProvider,omitempty"` // Optional: Embedding provider (voyage, openai, cohere, etc.)
+	Providers          map[string]RAGProviderConfig          `json:"providers,omitempty"`
+	EmbeddingProviders map[string]RAGEmbeddingProviderConfig `json:"embeddingProviders,omitempty"` // Embedding provider configs
 }
 
 // RAGProviderConfig contains RAG provider-specific settings
 // TODO: Refactor this to use a common interface for all RAG providers, can use environment variables to configure the different providers
 type RAGProviderConfig struct {
 	DatabasePath             string  `json:"databasePath,omitempty"`             // Simple provider: path to JSON database
-	IndexName                string  `json:"indexName,omitempty"`                // OpenAI provider: vector store name
+	IndexName                string  `json:"indexName,omitempty"`                // OpenAI/S3 provider: vector store/index name
 	VectorStoreID            string  `json:"vectorStoreId,omitempty"`            // OpenAI provider: existing vector store ID
 	Dimensions               int     `json:"dimensions,omitempty"`               // OpenAI provider: embedding dimensions
 	SimilarityMetric         string  `json:"similarityMetric,omitempty"`         // OpenAI provider: similarity metric
-	MaxResults               int     `json:"maxResults,omitempty"`               // OpenAI provider: maximum search results
-	ScoreThreshold           float64 `json:"scoreThreshold,omitempty"`           // OpenAI provider: score threshold
+	MaxResults               int     `json:"maxResults,omitempty"`               // OpenAI/S3 provider: maximum search results
+	ScoreThreshold           float64 `json:"scoreThreshold,omitempty"`           // OpenAI/S3 provider: score threshold
 	RewriteQuery             bool    `json:"rewriteQuery,omitempty"`             // OpenAI provider: rewrite query
 	VectorStoreNameRegex     string  `json:"vectorStoreNameRegex,omitempty"`     // OpenAI provider: vector store name regex
 	VectorStoreMetadataKey   string  `json:"vectorStoreMetadataKey,omitempty"`   // OpenAI provider: vector store metadata key
 	VectorStoreMetadataValue string  `json:"vectorStoreMetadataValue,omitempty"` // OpenAI provider: vector store metadata value
+	BucketName               string  `json:"bucketName,omitempty"`               // S3 provider: S3 bucket name
+	Region                   string  `json:"region,omitempty"`                   // S3 provider: AWS region
+	DateFilterField          string  `json:"dateFilterField,omitempty"`          // Date filter metadata field name
+	DateRangeWindowDays      int     `json:"dateRangeWindowDays,omitempty"`      // Days to expand date range backward (default: 7)
+}
+
+// RAGEmbeddingProviderConfig contains embedding provider-specific settings
+type RAGEmbeddingProviderConfig struct {
+	APIKey string `json:"apiKey,omitempty"` // API key for the embedding provider
 }
 
 // MonitoringConfig contains monitoring and observability settings

internal/config/validation.go

@@ -194,6 +194,12 @@ func (c *Config) SubstituteEnvironmentVariables() {
 	c.Observability.ServiceName = substituteEnvVars(c.Observability.ServiceName)
 	c.Observability.ServiceVersion = substituteEnvVars(c.Observability.ServiceVersion)
 
+	// Substitute in RAG Embedding Providers configuration
+	for name, provider := range c.RAG.EmbeddingProviders {
+		provider.APIKey = substituteEnvVars(provider.APIKey)
+		c.RAG.EmbeddingProviders[name] = provider
+	}
+
 }
 
 // substituteEnvVars replaces ${VAR_NAME} patterns with environment variable values

internal/handlers/llm_mcp_bridge.go

@@ -224,54 +224,62 @@ func NewLLMMCPBridgeFromClientsWithLogLevel(mcpClients interface{}, stdLogger *l
 	return NewLLMMCPBridgeWithLogLevel(interfaceClients, stdLogger, discoveredTools, logLevel, llmRegistry, cfg)
 }
 
-// ProcessLLMResponse processes an LLM response, expecting a specific JSON tool call format.
-// It no longer uses natural language detection.
-func (b *LLMMCPBridge) ProcessLLMResponse(ctx context.Context, llmResponse *llms.ContentChoice, _ string, extraArgs map[string]interface{}) (string, error) {
+// ExtractToolCall extracts tool call information from LLM response
+// Returns nil if no tool call is detected
+func (b *LLMMCPBridge) ExtractToolCall(llmResponse *llms.ContentChoice) (*ToolCall, error) {
 	var toolCall *ToolCall
 	var err error
+
+	// Check for native tool calls first
 	funcCall := llmResponse.FuncCall
-	// Check for a tool call in JSON format
 	if len(llmResponse.ToolCalls) > 0 {
 		funcCall = llmResponse.ToolCalls[0].FunctionCall
 	}
 
 	if funcCall != nil {
 		toolCall, err = b.getToolCall(funcCall)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 	} else {
+		// Fallback: try to detect JSON tool call in Content
 		toolCall = b.detectSpecificJSONToolCall(llmResponse.Content)
 	}
 
-	if toolCall != nil {
-		// Execute the tool call
-		result, err := b.executeToolCall(ctx, toolCall, extraArgs)
-		if err != nil {
-			// Check if it's already a domain error
-			var errorMessage string
-			if customErrors.IsDomainError(err) {
-				// Extract structured information from the domain error
-				code, _ := customErrors.GetErrorCode(err)
-				b.logger.ErrorKV("Failed to execute tool call",
-					"error", err.Error(),
-					"error_code", code,
-					"tool", toolCall.Tool)
-				errorMessage = fmt.Sprintf("Error executing tool call: %v (code: %s)", err, code)
-			} else {
-				b.logger.ErrorKV("Failed to execute tool call",
-					"error", err.Error(),
-					"tool", toolCall.Tool)
-				errorMessage = fmt.Sprintf("Error executing tool call: %v", err)
-			}
+	return toolCall, nil
+}
 
-			return errorMessage, nil
+// ExecuteToolCall executes a tool call and returns the result
+func (b *LLMMCPBridge) ExecuteToolCall(ctx context.Context, toolCall *ToolCall, extraArgs map[string]interface{}) (string, error) {
+	if toolCall == nil {
+		return "", fmt.Errorf("toolCall cannot be nil")
+	}
+
+	// Execute the tool call
+	result, err := b.executeToolCall(ctx, toolCall, extraArgs)
+	if err != nil {
+		// Check if it's already a domain error
+		var errorMessage string
+		if customErrors.IsDomainError(err) {
+			// Extract structured information from the domain error
+			code, _ := customErrors.GetErrorCode(err)
+			b.logger.ErrorKV("Failed to execute tool call",
+				"error", err.Error(),
+				"error_code", code,
+				"tool", toolCall.Tool)
+			errorMessage = err.Error()
+		} else {
+			// Wrap as domain error
+			domainErr := customErrors.WrapMCPError(err, "tool_execution_failed",
+				fmt.Sprintf("Failed to execute tool '%s'", toolCall.Tool))
+			b.logger.ErrorKV("Failed to execute tool call", "error", domainErr.Error(), "tool", toolCall.Tool)
+			errorMessage = domainErr.Error()
 		}
-		return result, nil
+		return "", fmt.Errorf("%s", errorMessage)
 	}
 
-	// Just return the LLM response as-is if no tool call was detected
-	return llmResponse.Content, nil
+	b.logger.DebugKV("Tool call executed successfully", "tool", toolCall.Tool, "result_length", len(result))
+	return result, nil
 }
 
 // ToolCall represents the expected JSON structure for a tool call from the LLM

internal/observability/langfuse.go

@@ -197,17 +197,23 @@ func (p *LangfuseProvider) SetOutput(span OtelTrace.Span, output string) {
 }
 
 func (p *LangfuseProvider) SetTokenUsage(span OtelTrace.Span, promptTokens, completionTokens, reasoningTokens, totalTokens int) {
-	// Langfuse usage format
+	// Langfuse usage format - uses "input", "output", "total" field names
+	// Map our standard token names to Langfuse's expected format
 	usageDetails := map[string]int{
-		"prompt_tokens":     promptTokens,
-		"completion_tokens": completionTokens,
-		"total_tokens":      totalTokens,
-		"reasoning_tokens":  reasoningTokens,
+		"input":  promptTokens,
+		"output": completionTokens,
+		"total":  totalTokens,
+	}
+
+	// Add reasoning tokens as a separate metadata field since Langfuse doesn't have a standard field for it
+	if reasoningTokens > 0 {
+		usageDetails["reasoning_tokens"] = reasoningTokens
 	}
 
 	if usageJSON, err := json.Marshal(usageDetails); err == nil {
 		span.SetAttributes(
 			attribute.String("langfuse.observation.usage_details", string(usageJSON)),
+			// Also set OpenTelemetry standard fields for compatibility
 			attribute.Int("llm.token_count.prompt_tokens", promptTokens),
 			attribute.Int("llm.token_count.completion_tokens", completionTokens),
 			attribute.Int("llm.token_count.total_tokens", totalTokens),