diff --git a/agent_mock.go b/agent_mock.go new file mode 100644 index 00000000..581726de --- /dev/null +++ b/agent_mock.go @@ -0,0 +1,353 @@ +package shuffle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/url" + "os" + "path/filepath" +) + +func RunAgentDecisionMockHandler(execution WorkflowExecution, decision AgentDecision) ([]byte, string, string, error) { + log.Printf("[DEBUG][%s] Mock handler called for tool=%s, action=%s", execution.ExecutionId, decision.Tool, decision.Action) + + // Get mock response + response, err := GetMockSingulResponse(execution.ExecutionId, decision.Fields) + if err != nil { + log.Printf("[ERROR][%s] Failed to get mock response: %s", execution.ExecutionId, err) + return nil, "", decision.Tool, err + } + + // Parse the response to extract raw_response + var outputMapped SchemalessOutput + err = json.Unmarshal(response, &outputMapped) + if err != nil { + log.Printf("[ERROR][%s] Failed to unmarshal mock response: %s", execution.ExecutionId, err) + return response, "", decision.Tool, err + } + + // Extract the raw_response field + body := response + if val, ok := outputMapped.RawResponse.(string); ok { + body = []byte(val) + } else if val, ok := outputMapped.RawResponse.([]byte); ok { + body = val + } else if val, ok := outputMapped.RawResponse.(map[string]interface{}); ok { + marshalledRawResp, err := json.MarshalIndent(val, "", " ") + if err != nil { + log.Printf("[ERROR][%s] Failed to marshal raw response: %s", execution.ExecutionId, err) + } else { + body = marshalledRawResp + } + } + + log.Printf("[DEBUG][%s] Returning mock response for %s (success=%v, response_size=%d bytes)", + execution.ExecutionId, decision.Tool, outputMapped.Success, len(body)) + + return body, "", decision.Tool, nil +} + +func GetMockSingulResponse(executionId string, fields []Valuereplace) ([]byte, error) { + ctx := context.Background() + mockCacheKey := fmt.Sprintf("agent_mock_%s", executionId) + cache, err := GetCache(ctx, mockCacheKey) + + if err == nil { + cacheData := cache.([]uint8) + log.Printf("[DEBUG][%s] Using cached mock data (%d bytes)", executionId, len(cacheData)) + + var toolCalls []MockToolCall + err = json.Unmarshal(cacheData, &toolCalls) + if err != nil { + log.Printf("[ERROR][%s] Failed to unmarshal cached mock data: %s", executionId, err) + return nil, fmt.Errorf("failed to unmarshal cached mock data: %w", err) + } + + return GetMockResponseFromToolCalls(toolCalls, fields) + } + + testDataPath := os.Getenv("AGENT_TEST_DATA_PATH") + if testDataPath == "" { + return nil, fmt.Errorf("no mock data in cache for execution %s and AGENT_TEST_DATA_PATH not set", executionId) + } + + log.Printf("[DEBUG][%s] Cache miss, using file-based mocks from: %s", executionId, testDataPath) + + useCase := os.Getenv("AGENT_TEST_USE_CASE") + if useCase == "" { + return nil, errors.New("AGENT_TEST_USE_CASE not set") + } + + useCaseData, err := loadUseCaseData(useCase) + if err != nil { + return nil, err + } + + return GetMockResponseFromToolCalls(useCaseData.ToolCalls, fields) +} + +// GetMockResponseFromToolCalls finds and returns the matching mock response from tool calls +func GetMockResponseFromToolCalls(toolCalls []MockToolCall, fields []Valuereplace) ([]byte, error) { + requestURL := extractFieldValue(fields, "url") + if requestURL == "" { + return nil, errors.New("no URL found in request fields") + } + + log.Printf("[DEBUG] Looking for mock data with URL: %s", requestURL) + + var candidates []MockToolCall + reqURLParsed, err := url.Parse(requestURL) + if err != nil { + log.Printf("[ERROR] Invalid request URL %s: %v", requestURL, err) + return nil, fmt.Errorf("invalid request URL: %w", err) + } + for _, tc := range toolCalls { + if urlsEqual(reqURLParsed, tc.URL) { + candidates = append(candidates, tc) + } + } + + // If no exact matches, try fuzzy matching + if len(candidates) == 0 { + log.Printf("[DEBUG] No exact match, trying fuzzy matching...") + bestMatch, score := findBestFuzzyMatch(reqURLParsed, toolCalls) + if score >= 0.80 { + log.Printf("[INFO] Found fuzzy match with %.1f%% similarity: %s", score*100, bestMatch.URL) + candidates = append(candidates, bestMatch) + } else { + return nil, fmt.Errorf("no mock data found for URL: %s (best match: %.1f%%)", requestURL, score*100) + } + } + + // If only one match, return it + if len(candidates) == 1 { + log.Printf("[DEBUG] Found exact match for URL: %s", requestURL) + return marshalResponse(candidates[0].Response) + } + + // Multiple matches - compare fields to find exact match + log.Printf("[DEBUG] Found %d candidates for URL, comparing fields...", len(candidates)) + for _, candidate := range candidates { + if fieldsMatch(fields, candidate.Fields) { + log.Printf("[DEBUG] Found exact match based on fields") + return marshalResponse(candidate.Response) + } + } + + // No exact match - return first candidate with a warning + log.Printf("[WARNING] No exact field match found, returning first candidate") + return marshalResponse(candidates[0].Response) +} + +func urlsEqual(req *url.URL, stored string) bool { + storedURL, err := url.Parse(stored) + if err != nil { + log.Printf("[WARN] Invalid stored URL %s: %v", stored, err) + return false + } + if req.Scheme != storedURL.Scheme || req.Host != storedURL.Host || req.Path != storedURL.Path { + return false + } + reqQuery := req.Query() + storedQuery := storedURL.Query() + // If the number of parameters differs, not a match + if len(reqQuery) != len(storedQuery) { + return false + } + + for key, reqVals := range reqQuery { + storedVals, ok := storedQuery[key] + if !ok { + return false + } + if len(reqVals) != len(storedVals) { + return false + } + for i, v := range reqVals { + if v != storedVals[i] { + return false + } + } + } + return true +} + +func loadUseCaseData(useCase string) (*MockUseCaseData, error) { + possiblePaths := []string{} + + if envPath := os.Getenv("AGENT_TEST_DATA_PATH"); envPath != "" { + possiblePaths = append(possiblePaths, envPath) + } + + possiblePaths = append(possiblePaths, "agent_test_data") + possiblePaths = append(possiblePaths, "../shuffle-shared/agent_test_data") + possiblePaths = append(possiblePaths, "../../shuffle-shared/agent_test_data") + + if homeDir, err := os.UserHomeDir(); err == nil { + possiblePaths = append(possiblePaths, filepath.Join(homeDir, "Documents", "shuffle-shared", "agent_test_data")) + } + + var filePath string + var foundPath string + + for _, basePath := range possiblePaths { + testPath := filepath.Join(basePath, fmt.Sprintf("%s.json", useCase)) + if _, err := os.Stat(testPath); err == nil { + filePath = testPath + foundPath = basePath + break + } + } + + if filePath == "" { + return nil, fmt.Errorf("could not find test data file %s.json in any of these paths: %v", useCase, possiblePaths) + } + + log.Printf("[DEBUG] Loading use case data from: %s", filePath) + + data, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read use case file %s: %s", filePath, err) + } + + var useCaseData MockUseCaseData + err = json.Unmarshal(data, &useCaseData) + if err != nil { + return nil, fmt.Errorf("failed to parse use case data: %s", err) + } + + log.Printf("[DEBUG] Loaded use case '%s' with %d tool calls from %s", useCaseData.UseCase, len(useCaseData.ToolCalls), foundPath) + + return &useCaseData, nil +} + +func extractFieldValue(fields []Valuereplace, key string) string { + for _, field := range fields { + if field.Key == key { + return field.Value + } + } + return "" +} + +func fieldsMatch(requestFields []Valuereplace, storedFields map[string]string) bool { + // Convert request fields to map for easier comparison + requestMap := make(map[string]string) + for _, field := range requestFields { + requestMap[field.Key] = field.Value + } + + for key, storedValue := range storedFields { + requestValue, exists := requestMap[key] + if !exists || requestValue != storedValue { + return false + } + } + + return true +} + +func marshalResponse(response map[string]interface{}) ([]byte, error) { + data, err := json.Marshal(response) + if err != nil { + return nil, fmt.Errorf("failed to marshal response: %s", err) + } + return data, nil +} + +func findBestFuzzyMatch(reqURL *url.URL, toolCalls []MockToolCall) (MockToolCall, float64) { + var bestMatch MockToolCall + bestScore := 0.0 + + for _, tc := range toolCalls { + storedURL, err := url.Parse(tc.URL) + if err != nil { + continue + } + + score := calculateURLSimilarity(reqURL, storedURL) + if score > bestScore { + bestScore = score + bestMatch = tc + } + } + + return bestMatch, bestScore +} + +func calculateURLSimilarity(url1, url2 *url.URL) float64 { + score := 0.0 + totalWeight := 0.0 + + // Scheme (10% weight) + if url1.Scheme == url2.Scheme { + score += 0.10 + } + totalWeight += 0.10 + + // Host (20% weight) + if url1.Host == url2.Host { + score += 0.20 + } + totalWeight += 0.20 + + // Path (20% weight) + if url1.Path == url2.Path { + score += 0.20 + } + totalWeight += 0.20 + + // Query parameters (50% weight) + query1 := url1.Query() + query2 := url2.Query() + + if len(query1) == 0 && len(query2) == 0 { + score += 0.50 + } else if len(query1) > 0 || len(query2) > 0 { + matchingParams := 0 + totalParams := 0 + + allKeys := make(map[string]bool) + for k := range query1 { + allKeys[k] = true + } + for k := range query2 { + allKeys[k] = true + } + totalParams = len(allKeys) + + // Count how many match + for key := range allKeys { + val1, ok1 := query1[key] + val2, ok2 := query2[key] + + if ok1 && ok2 { + // Both have this key - check if values match + if len(val1) == len(val2) { + allMatch := true + for i := range val1 { + if val1[i] != val2[i] { + allMatch = false + break + } + } + if allMatch { + matchingParams++ + } + } + } + } + + if totalParams > 0 { + paramScore := float64(matchingParams) / float64(totalParams) + score += paramScore * 0.50 + } + } + totalWeight += 0.50 + + return score / totalWeight +} \ No newline at end of file diff --git a/ai.go b/ai.go index 0ad9500e..4fe433b3 100644 --- a/ai.go +++ b/ai.go @@ -11973,4 +11973,4 @@ func buildManualInputList(history []ConversationMessage, newPrompt string) []map }) return items -} +} \ No newline at end of file diff --git a/cloudSync.go b/cloudSync.go index 23e07bd6..ac3800bc 100755 --- a/cloudSync.go +++ b/cloudSync.go @@ -2109,6 +2109,14 @@ func RunAgentDecisionSingulActionHandler(execution WorkflowExecution, decision A debugUrl := "" log.Printf("[INFO][%s] Running agent decision action '%s' with app '%s'. This is ran with Singul.", execution.ExecutionId, decision.Action, decision.Tool) + // Check if running in test mode + if os.Getenv("AGENT_TEST_MODE") == "true" { + log.Printf("[DEBUG][%s] AGENT_TEST_MODE enabled - using mock tool execution", execution.ExecutionId) + + // Call mock handler + return RunAgentDecisionMockHandler(execution, decision) + } + baseUrl := "https://shuffler.io" if os.Getenv("BASE_URL") != "" { baseUrl = os.Getenv("BASE_URL") diff --git a/shared.go b/shared.go index 28845bf5..5fe6a118 100755 --- a/shared.go +++ b/shared.go @@ -16278,8 +16278,43 @@ func handleAgentDecisionStreamResult(workflowExecution WorkflowExecution, action } if foundActionResultIndex < 0 { - log.Printf("[ERROR][%s] Action '%s' was NOT found with any result in the execution (yet)", workflowExecution.ExecutionId, actionResult.Action.ID) - return &workflowExecution, false, errors.New(fmt.Sprintf("ActionResultIndex: Agent node ID for decision ID %s not found", decisionId)) + // In test mode, Singul doesn't create sub-executions, so we need to handle this gracefully + if os.Getenv("AGENT_TEST_MODE") == "true" { + log.Printf("[DEBUG][%s] AGENT_TEST_MODE: Action '%s' not found in results, creating placeholder", workflowExecution.ExecutionId, actionResult.Action.ID) + + // Try to get the initial agent output from cache + ctx := context.Background() + actionCacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID) + placeholderResult := `{"status":"RUNNING","decisions":[]}` + + cache, err := GetCache(ctx, actionCacheId) + if err == nil { + // Found cached agent output - use it! + cacheData := []byte(cache.([]uint8)) + log.Printf("[DEBUG][%s] Found cached agent output for placeholder (size: %d bytes)", workflowExecution.ExecutionId, len(cacheData)) + placeholderResult = string(cacheData) + } else { + log.Printf("[DEBUG][%s] No cached agent output found, using empty placeholder", workflowExecution.ExecutionId) + } + + // Create a placeholder result for the agent action + placeholder := ActionResult{ + Action: actionResult.Action, + ExecutionId: workflowExecution.ExecutionId, + Result: placeholderResult, + StartedAt: time.Now().Unix(), + CompletedAt: 0, + Status: "EXECUTING", + } + + workflowExecution.Results = append(workflowExecution.Results, placeholder) + foundActionResultIndex = len(workflowExecution.Results) - 1 + + log.Printf("[DEBUG][%s] Created placeholder result at index %d", workflowExecution.ExecutionId, foundActionResultIndex) + } else { + log.Printf("[ERROR][%s] Action '%s' was NOT found with any result in the execution (yet)", workflowExecution.ExecutionId, actionResult.Action.ID) + return &workflowExecution, false, errors.New(fmt.Sprintf("ActionResultIndex: Agent node ID for decision ID %s not found", decisionId)) + } } mappedResult := AgentOutput{} @@ -16291,6 +16326,28 @@ func handleAgentDecisionStreamResult(workflowExecution WorkflowExecution, action return &workflowExecution, false, err } + // In test mode, if the placeholder has no decisions, we need to add the incoming decision + if os.Getenv("AGENT_TEST_MODE") == "true" && len(mappedResult.Decisions) == 0 { + log.Printf("[DEBUG][%s] AGENT_TEST_MODE: Placeholder has no decisions, parsing incoming decision", workflowExecution.ExecutionId) + + // Parse the incoming decision from actionResult + incomingDecision := AgentDecision{} + err = json.Unmarshal([]byte(actionResult.Result), &incomingDecision) + if err != nil { + log.Printf("[ERROR][%s] Failed unmarshalling incoming decision: %s", workflowExecution.ExecutionId, err) + } else { + // Add the decision to the mapped result + mappedResult.Decisions = append(mappedResult.Decisions, incomingDecision) + mappedResult.Status = "RUNNING" + + // Update the workflow execution result with the new decision + updatedResult, _ := json.Marshal(mappedResult) + workflowExecution.Results[foundActionResultIndex].Result = string(updatedResult) + + log.Printf("[DEBUG][%s] Added decision %s to placeholder (total decisions: %d)", workflowExecution.ExecutionId, incomingDecision.RunDetails.Id, len(mappedResult.Decisions)) + } + } + // FIXME: Need to check the current value from the workflowexecution here, instead of using the currently sent in decision // 1. Get the current result for the action @@ -20345,6 +20402,23 @@ func PrepareSingleAction(ctx context.Context, user User, appId string, body []by SetWorkflowExecution(ctx, exec, true) + if os.Getenv("AGENT_TEST_MODE") == "true" { + var bodyMap map[string]interface{} + if err := json.Unmarshal(body, &bodyMap); err == nil { + if mockToolCalls, ok := bodyMap["mock_tool_calls"]; ok { + mockCacheKey := fmt.Sprintf("agent_mock_%s", exec.ExecutionId) + mockData, _ := json.Marshal(mockToolCalls) + err := SetCache(ctx, mockCacheKey, mockData, 10) + if err != nil { + log.Printf("[ERROR] Failed to set cache the mock_tool_calls data for the exection %s", exec.ExecutionId) + } + log.Printf("[DEBUG] Cached mock tool calls for execution %s", exec.ExecutionId) + } else { + log.Printf("[WARNING] No mock_tool_calls found in the request body") + } + } + } + action, err := HandleAiAgentExecutionStart(exec, action, false) if err != nil { log.Printf("[ERROR] Failed to handle AI agent execution start: %s", err) diff --git a/structs.go b/structs.go index f48170b0..c8b7bc16 100755 --- a/structs.go +++ b/structs.go @@ -4834,3 +4834,58 @@ type StreamData struct { Chunk string `json:"chunk,omitempty"` Data string `json:"data,omitempty"` // For the final ID or error } + +type MockToolCall struct { + URL string `json:"url"` + Method string `json:"method"` + Fields map[string]string `json:"fields"` + Response map[string]interface{} `json:"response"` +} + +type MockUseCaseData struct { + UseCase string `json:"use_case"` + UserPrompt string `json:"user_prompt"` + ToolCalls []MockToolCall `json:"tool_calls"` + ExpectedDecisions []AgentDecision `json:"expected_decisions"` +} + +type AgentStartResponse struct { + Success bool `json:"success"` + ExecutionId string `json:"execution_id"` + Authorization string `json:"authorization"` +} + +type StreamsResultResponse struct { + Result string `json:"result"` + Results []ActionResult `json:"results"` + Status string `json:"status"` +} + +type AgentStartRequest struct { + ID string `json:"id"` + Name string `json:"name"` + AppName string `json:"app_name"` + AppID string `json:"app_id"` + AppVersion string `json:"app_version"` + Environment string `json:"environment"` + Parameters []map[string]string `json:"parameters"` +} + +type StreamsResultRequest struct { + ExecutionID string `json:"execution_id"` + Authorization string `json:"authorization"` +} + +type TestResponse struct { + Success bool `json:"success"` + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Results []TestResult `json:"results"` +} + +type TestResult struct { + TestCase string `json:"test_case"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} \ No newline at end of file