Improves and standardizes Flakeguard summary data

tools/flakeguard/Makefile

@@ -30,17 +30,17 @@ example:
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
 	go run . aggregate-results \
-      --results-path ./example_results \
-      --output-path ./example_results \
-      --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
-	  --branch-name "example-branch" \
-      --base-sha "abc" \
-      --head-sha "xyz" \
-      --github-workflow-name "ExampleWorkflowName" \
-	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
-	  --splunk-url "https://splunk.example.com" \
-	  --splunk-token "splunk-token" \
-	  --splunk-event "example-splunk-event"
+		--results-path ./example_results \
+		--output-path ./example_results \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--branch-name "example-branch" \
+		--base-sha "abc" \
+		--head-sha "xyz" \
+		--github-workflow-name "ExampleWorkflowName" \
+		--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+		--splunk-url "https://splunk.example.com" \
+		--splunk-token "splunk-token" \
+		--splunk-event "example-splunk-event"
 
 .PHONY: example_flaky_panic
 example_flaky_panic:
@@ -50,17 +50,17 @@ example_flaky_panic:
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
 	go run . aggregate-results \
-	  --results-path ./example_results \
-	  --output-path ./example_results \
-	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
-	  --branch-name "example-branch" \
-      --base-sha "abc" \
-      --head-sha "xyz" \
-      --github-workflow-name "ExampleWorkflowName" \
-	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
-	  --splunk-url "https://splunk.example.com" \
-	  --splunk-token "splunk-token" \
-	  --splunk-event "example-splunk-event"
+		--results-path ./example_results \
+		--output-path ./example_results \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--branch-name "example-branch" \
+		--base-sha "abc" \
+		--head-sha "xyz" \
+		--github-workflow-name "ExampleWorkflowName" \
+		--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+		--splunk-url "https://splunk.example.com" \
+		--splunk-token "splunk-token" \
+		--splunk-event "example-splunk-event"
 
 .PHONY: example_timeout
 example_timeout:
@@ -70,14 +70,14 @@ example_timeout:
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
 	go run . aggregate-results \
-	  --results-path ./example_results \
-	  --output-path ./example_results \
-	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
-	  --branch-name "example-branch" \
-      --base-sha "abc" \
-      --head-sha "xyz" \
-      --github-workflow-name "ExampleWorkflowName" \
-	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
-	  --splunk-url "https://splunk.example.com" \
-	  --splunk-token "splunk-token" \
-	  --splunk-event "example-splunk-event"
+		--results-path ./example_results \
+		--output-path ./example_results \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--branch-name "example-branch" \
+		--base-sha "abc" \
+		--head-sha "xyz" \
+		--github-workflow-name "ExampleWorkflowName" \
+		--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+		--splunk-url "https://splunk.example.com" \
+		--splunk-token "splunk-token" \
+		--splunk-event "example-splunk-event"

tools/flakeguard/cmd/aggregate_results.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -22,7 +21,6 @@ var AggregateResultsCmd = &cobra.Command{
 		// Get flag values
 		resultsPath, _ := cmd.Flags().GetString("results-path")
 		outputDir, _ := cmd.Flags().GetString("output-path")
-		summaryFileName, _ := cmd.Flags().GetString("summary-file-name")
 		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		codeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
 		repoPath, _ := cmd.Flags().GetString("repo-path")
@@ -47,6 +45,7 @@ var AggregateResultsCmd = &cobra.Command{
 		s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 		s.Suffix = " Aggregating test reports..."
 		s.Start()
+		fmt.Println()
 
 		// Load test reports from JSON files and aggregate them
 		aggregatedReport, err := reports.LoadAndAggregate(
@@ -62,12 +61,10 @@ var AggregateResultsCmd = &cobra.Command{
 		)
 		if err != nil {
 			s.Stop()
-			fmt.Println()
-			log.Error().Err(err).Msg("Error aggregating test reports")
+			log.Error().Err(err).Stack().Msg("Error aggregating test reports")
 			os.Exit(ErrorExitCode)
 		}
 		s.Stop()
-		fmt.Println()
 		log.Debug().Msg("Successfully loaded and aggregated test reports")
 
 		// Start spinner for mapping test results to paths
@@ -79,37 +76,33 @@ var AggregateResultsCmd = &cobra.Command{
 		err = reports.MapTestResultsToPaths(aggregatedReport, repoPath)
 		if err != nil {
 			s.Stop()
-			fmt.Println()
-			log.Error().Err(err).Msg("Error mapping test results to paths")
+			log.Error().Stack().Err(err).Msg("Error mapping test results to paths")
 			os.Exit(ErrorExitCode)
 		}
 		s.Stop()
-		fmt.Println()
 		log.Debug().Msg("Successfully mapped paths to test results")
 
 		// Map test results to code owners if codeOwnersPath is provided
 		if codeOwnersPath != "" {
 			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 			s.Suffix = " Mapping test results to code owners..."
 			s.Start()
+			fmt.Println()
 
 			err = reports.MapTestResultsToOwners(aggregatedReport, codeOwnersPath)
 			if err != nil {
 				s.Stop()
-				fmt.Println()
-				log.Error().Err(err).Msg("Error mapping test results to code owners")
+				log.Error().Stack().Err(err).Msg("Error mapping test results to code owners")
 				os.Exit(ErrorExitCode)
 			}
 			s.Stop()
-			fmt.Println()
 			log.Debug().Msg("Successfully mapped code owners to test results")
 		}
 
 		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
 			return !tr.Skipped && tr.PassRatio < maxPassRatio
 		})
 		s.Stop()
-		fmt.Println()
 
 		// Check if there are any failed tests
 		if len(failedTests) > 0 {
@@ -118,7 +111,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Create a new report for failed tests with logs
 			failedReportWithLogs := &reports.TestReport{
 				GoProject:          aggregatedReport.GoProject,
-				TestRunCount:       aggregatedReport.TestRunCount,
+				SummaryData:        aggregatedReport.SummaryData,
 				RaceDetection:      aggregatedReport.RaceDetection,
 				ExcludedTests:      aggregatedReport.ExcludedTests,
 				SelectedTests:      aggregatedReport.SelectedTests,
@@ -131,7 +124,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Save the failed tests report with logs
 			failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
 			if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
-				log.Error().Err(err).Msg("Error saving failed tests report with logs")
+				log.Error().Stack().Err(err).Msg("Error saving failed tests report with logs")
 				os.Exit(ErrorExitCode)
 			}
 			log.Debug().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")
@@ -146,7 +139,7 @@ var AggregateResultsCmd = &cobra.Command{
 			// Save the failed tests report without logs
 			failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-results.json")
 			if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportWithLogs); err != nil {
-				log.Error().Err(err).Msg("Error saving failed tests report without logs")
+				log.Error().Stack().Err(err).Msg("Error saving failed tests report without logs")
 				os.Exit(ErrorExitCode)
 			}
 			log.Debug().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
@@ -164,39 +157,16 @@ var AggregateResultsCmd = &cobra.Command{
 		// Save the aggregated report to the output directory
 		aggregatedReportPath := filepath.Join(outputDir, "all-test-results.json")
 		if err := reports.SaveReport(fs, aggregatedReportPath, *aggregatedReport); err != nil {
-			log.Error().Err(err).Msg("Error saving aggregated test report")
+			log.Error().Stack().Err(err).Msg("Error saving aggregated test report")
 			os.Exit(ErrorExitCode)
 		}
-		log.Debug().Str("path", aggregatedReportPath).Msg("Aggregated test report saved")
-
-		// Generate all-tests-summary.json
-		var summaryFilePath string
-		if summaryFileName != "" {
-			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
-			s.Suffix = " Generating summary json..."
-			s.Start()
-
-			summaryFilePath = filepath.Join(outputDir, summaryFileName)
-			err = generateAllTestsSummaryJSON(aggregatedReport, summaryFilePath, maxPassRatio)
-			if err != nil {
-				s.Stop()
-				fmt.Println()
-				log.Error().Err(err).Msg("Error generating summary json")
-				os.Exit(ErrorExitCode)
-			}
-			s.Stop()
-			fmt.Println()
-			log.Debug().Str("path", summaryFilePath).Msg("Summary generated")
-		}
-
-		log.Info().Str("summary", summaryFilePath).Str("report", aggregatedReportPath).Msg("Aggregation complete")
+		log.Info().Str("report", aggregatedReportPath).Msg("Aggregation complete")
 	},
 }
 
 func init() {
 	AggregateResultsCmd.Flags().StringP("results-path", "p", "", "Path to the folder containing JSON test result files (required)")
 	AggregateResultsCmd.Flags().StringP("output-path", "o", "./report", "Path to output the aggregated results (directory)")
-	AggregateResultsCmd.Flags().StringP("summary-file-name", "s", "all-test-summary.json", "Name of the summary JSON file")
 	AggregateResultsCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
 	AggregateResultsCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
 	AggregateResultsCmd.Flags().StringP("repo-path", "", ".", "The path to the root of the repository/project")
@@ -215,26 +185,3 @@ func init() {
 		log.Fatal().Err(err).Msg("Error marking flag as required")
 	}
 }
-
-// New function to generate all-tests-summary.json
-func generateAllTestsSummaryJSON(report *reports.TestReport, outputPath string, maxPassRatio float64) error {
-	summary := reports.GenerateSummaryData(report.Results, maxPassRatio)
-	data, err := json.Marshal(summary)
-	if err != nil {
-		return fmt.Errorf("error marshaling summary data to JSON: %w", err)
-	}
-
-	fs := reports.OSFileSystem{}
-	jsonFile, err := fs.Create(outputPath)
-	if err != nil {
-		return fmt.Errorf("error creating file: %w", err)
-	}
-	defer jsonFile.Close()
-
-	_, err = jsonFile.Write(data)
-	if err != nil {
-		return fmt.Errorf("error writing data to file: %w", err)
-	}
-
-	return nil
-}

tools/flakeguard/cmd/generate_report.go

@@ -17,29 +17,14 @@ import (
 	"golang.org/x/oauth2"
 )
 
-type SummaryData struct {
-	TotalTests     int     `json:"total_tests"`
-	PanickedTests  int     `json:"panicked_tests"`
-	RacedTests     int     `json:"raced_tests"`
-	FlakyTests     int     `json:"flaky_tests"`
-	FlakyTestRatio string  `json:"flaky_test_ratio"`
-	TotalRuns      int     `json:"total_runs"`
-	PassedRuns     int     `json:"passed_runs"`
-	FailedRuns     int     `json:"failed_runs"`
-	SkippedRuns    int     `json:"skipped_runs"`
-	PassRatio      string  `json:"pass_ratio"`
-	MaxPassRatio   float64 `json:"max_pass_ratio"`
-}
-
 var GenerateReportCmd = &cobra.Command{
 	Use:   "generate-report",
-	Short: "Generate reports from an aggregated test results",
+	Short: "Generate test reports from aggregated results that can be posted to GitHub",
 	Run: func(cmd *cobra.Command, args []string) {
 		fs := reports.OSFileSystem{}
 
 		// Get flag values
 		aggregatedResultsPath, _ := cmd.Flags().GetString("aggregated-results-path")
-		summaryPath, _ := cmd.Flags().GetString("summary-path")
 		outputDir, _ := cmd.Flags().GetString("output-path")
 		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		generatePRComment, _ := cmd.Flags().GetBool("generate-pr-comment")
@@ -79,28 +64,8 @@ var GenerateReportCmd = &cobra.Command{
 		fmt.Println()
 		log.Info().Msg("Successfully loaded aggregated test report")
 
-		// Load the summary data to check for failed tests
-		var summaryData SummaryData
-
-		if summaryPath == "" {
-			log.Error().Msg("Summary path is required")
-			os.Exit(ErrorExitCode)
-		}
-
-		summaryFile, err := os.Open(summaryPath)
-		if err != nil {
-			log.Error().Err(err).Msg("Error opening summary JSON file")
-			os.Exit(ErrorExitCode)
-		}
-		defer summaryFile.Close()
-
-		if err := json.NewDecoder(summaryFile).Decode(&summaryData); err != nil {
-			log.Error().Err(err).Msg("Error decoding summary JSON file")
-			os.Exit(ErrorExitCode)
-		}
-
 		// Check if there are failed tests
-		hasFailedTests := summaryData.FailedRuns > 0
+		hasFailedTests := aggregatedReport.SummaryData.FailedRuns > 0
 
 		var artifactLink string
 		if hasFailedTests {
@@ -199,7 +164,6 @@ var GenerateReportCmd = &cobra.Command{
 
 func init() {
 	GenerateReportCmd.Flags().StringP("aggregated-results-path", "i", "", "Path to the aggregated JSON report file (required)")
-	GenerateReportCmd.Flags().StringP("summary-path", "s", "", "Path to the summary JSON file (required)")
 	GenerateReportCmd.Flags().StringP("output-path", "o", "./report", "Path to output the generated report files")
 	GenerateReportCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
 	GenerateReportCmd.Flags().Bool("generate-pr-comment", false, "Set to true to generate PR comment markdown")
@@ -216,10 +180,6 @@ func init() {
 		log.Error().Err(err).Msg("Error marking flag as required")
 		os.Exit(ErrorExitCode)
 	}
-	if err := GenerateReportCmd.MarkFlagRequired("summary-path"); err != nil {
-		log.Error().Err(err).Msg("Error marking flag as required")
-		os.Exit(ErrorExitCode)
-	}
 	if err := GenerateReportCmd.MarkFlagRequired("github-repository"); err != nil {
 		log.Error().Err(err).Msg("Error marking flag as required")
 		os.Exit(ErrorExitCode)

tools/flakeguard/cmd/run.go

@@ -40,6 +40,11 @@ var RunTestsCmd = &cobra.Command{
 		shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
 		omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
 
+		if maxPassRatio < 0 || maxPassRatio > 1 {
+			log.Error().Float64("max pass ratio", maxPassRatio).Msg("Error: max pass ratio must be between 0 and 1")
+			os.Exit(ErrorExitCode)
+		}
+
 		// Check if project dependencies are correctly set up
 		if err := checkDependencies(projectPath); err != nil {
 			log.Error().Err(err).Msg("Error checking project dependencies")
@@ -74,6 +79,7 @@ var RunTestsCmd = &cobra.Command{
 			UseShuffle:           useShuffle,
 			ShuffleSeed:          shuffleSeed,
 			OmitOutputsOnSuccess: omitOutputsOnSuccess,
+			MaxPassRatio:         maxPassRatio,
 		}
 
 		// Run the tests
@@ -109,8 +115,14 @@ var RunTestsCmd = &cobra.Command{
 
 		if len(flakyTests) > 0 {
 			log.Info().Int("count", len(flakyTests)).Str("pass ratio threshold", fmt.Sprintf("%.2f%%", maxPassRatio*100)).Msg("Found flaky tests")
-			fmt.Printf("\nFlakeguard Summary\n")
-			reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false, false)
+		} else {
+			log.Info().Msg("No flaky tests found")
+		}
+
+		fmt.Printf("\nFlakeguard Summary\n")
+		reports.RenderResults(os.Stdout, testReport, false, false)
+
+		if len(flakyTests) > 0 {
 			// Exit with error code if there are flaky tests
 			os.Exit(FlakyTestsExitCode)
 		}

tools/flakeguard/go.mod

@@ -1,6 +1,6 @@
 module github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard
 
-go 1.23.4
+go 1.23.6
 
 require (
 	github.com/briandowns/spinner v1.23.1
@@ -21,6 +21,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.27.0 // indirect

tools/flakeguard/go.sum

@@ -25,6 +25,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=