Skip to content

Commit f6ff5f3

Browse files
kalverrakalverra
committed
Fixes and improves test checks
1 parent 60ea174 commit f6ff5f3

File tree

13 files changed

+1377
-1502
lines changed

13 files changed

+1377
-1502
lines changed

‎tools/flakeguard/Makefile

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,17 @@ example:
3030
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
3131
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
3232
go run . aggregate-results \
33-
--results-path ./example_results \
34-
--output-path ./example_results \
35-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
36-
--branch-name "example-branch" \
37-
--base-sha "abc" \
38-
--head-sha "xyz" \
39-
--github-workflow-name "ExampleWorkflowName" \
40-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
41-
--splunk-url "https://splunk.example.com" \
42-
--splunk-token "splunk-token" \
43-
--splunk-event "example-splunk-event"
33+
--results-path ./example_results \
34+
--output-path ./example_results \
35+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
36+
--branch-name "example-branch" \
37+
--base-sha "abc" \
38+
--head-sha "xyz" \
39+
--github-workflow-name "ExampleWorkflowName" \
40+
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
41+
--splunk-url "https://splunk.example.com" \
42+
--splunk-token "splunk-token" \
43+
--splunk-event "example-splunk-event"
4444

4545
.PHONY: example_flaky_panic
4646
example_flaky_panic:
@@ -50,17 +50,17 @@ example_flaky_panic:
5050
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
5151
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
5252
go run . aggregate-results \
53-
--results-path ./example_results \
54-
--output-path ./example_results \
55-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
56-
--branch-name "example-branch" \
57-
--base-sha "abc" \
58-
--head-sha "xyz" \
59-
--github-workflow-name "ExampleWorkflowName" \
60-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
61-
--splunk-url "https://splunk.example.com" \
62-
--splunk-token "splunk-token" \
63-
--splunk-event "example-splunk-event"
53+
--results-path ./example_results \
54+
--output-path ./example_results \
55+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
56+
--branch-name "example-branch" \
57+
--base-sha "abc" \
58+
--head-sha "xyz" \
59+
--github-workflow-name "ExampleWorkflowName" \
60+
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
61+
--splunk-url "https://splunk.example.com" \
62+
--splunk-token "splunk-token" \
63+
--splunk-event "example-splunk-event"
6464

6565
.PHONY: example_timeout
6666
example_timeout:
@@ -70,14 +70,14 @@ example_timeout:
7070
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
7171
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
7272
go run . aggregate-results \
73-
--results-path ./example_results \
74-
--output-path ./example_results \
75-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
76-
--branch-name "example-branch" \
77-
--base-sha "abc" \
78-
--head-sha "xyz" \
79-
--github-workflow-name "ExampleWorkflowName" \
80-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
81-
--splunk-url "https://splunk.example.com" \
82-
--splunk-token "splunk-token" \
83-
--splunk-event "example-splunk-event"
73+
--results-path ./example_results \
74+
--output-path ./example_results \
75+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
76+
--branch-name "example-branch" \
77+
--base-sha "abc" \
78+
--head-sha "xyz" \
79+
--github-workflow-name "ExampleWorkflowName" \
80+
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
81+
--splunk-url "https://splunk.example.com" \
82+
--splunk-token "splunk-token" \
83+
--splunk-event "example-splunk-event"

‎tools/flakeguard/cmd/aggregate_results.go

Lines changed: 1 addition & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package cmd
22

33
import (
4-
"encoding/json"
54
"fmt"
65
"os"
76
"path/filepath"
@@ -22,7 +21,6 @@ var AggregateResultsCmd = &cobra.Command{
2221
// Get flag values
2322
resultsPath, _ := cmd.Flags().GetString("results-path")
2423
outputDir, _ := cmd.Flags().GetString("output-path")
25-
summaryFileName, _ := cmd.Flags().GetString("summary-file-name")
2624
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
2725
codeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
2826
repoPath, _ := cmd.Flags().GetString("repo-path")
@@ -162,36 +160,13 @@ var AggregateResultsCmd = &cobra.Command{
162160
log.Error().Stack().Err(err).Msg("Error saving aggregated test report")
163161
os.Exit(ErrorExitCode)
164162
}
165-
log.Debug().Str("path", aggregatedReportPath).Msg("Aggregated test report saved")
166-
167-
// Generate all-tests-summary.json
168-
var summaryFilePath string
169-
if summaryFileName != "" {
170-
s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
171-
s.Suffix = " Generating summary json..."
172-
s.Start()
173-
fmt.Println()
174-
175-
summaryFilePath = filepath.Join(outputDir, summaryFileName)
176-
err = generateAllTestsSummaryJSON(aggregatedReport, summaryFilePath, maxPassRatio)
177-
if err != nil {
178-
s.Stop()
179-
fmt.Println()
180-
log.Error().Stack().Err(err).Msg("Error generating summary json")
181-
os.Exit(ErrorExitCode)
182-
}
183-
s.Stop()
184-
log.Debug().Str("path", summaryFilePath).Msg("Summary generated")
185-
}
186-
187-
log.Info().Str("summary", summaryFilePath).Str("report", aggregatedReportPath).Msg("Aggregation complete")
163+
log.Info().Str("report", aggregatedReportPath).Msg("Aggregation complete")
188164
},
189165
}
190166

191167
func init() {
192168
AggregateResultsCmd.Flags().StringP("results-path", "p", "", "Path to the folder containing JSON test result files (required)")
193169
AggregateResultsCmd.Flags().StringP("output-path", "o", "./report", "Path to output the aggregated results (directory)")
194-
AggregateResultsCmd.Flags().StringP("summary-file-name", "s", "all-test-summary.json", "Name of the summary JSON file")
195170
AggregateResultsCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
196171
AggregateResultsCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
197172
AggregateResultsCmd.Flags().StringP("repo-path", "", ".", "The path to the root of the repository/project")
@@ -210,26 +185,3 @@ func init() {
210185
log.Fatal().Err(err).Msg("Error marking flag as required")
211186
}
212187
}
213-
214-
// New function to generate all-tests-summary.json
215-
func generateAllTestsSummaryJSON(report *reports.TestReport, outputPath string, maxPassRatio float64) error {
216-
summary := reports.GenerateSummaryData(report.Results, maxPassRatio)
217-
data, err := json.Marshal(summary)
218-
if err != nil {
219-
return fmt.Errorf("error marshaling summary data to JSON: %w", err)
220-
}
221-
222-
fs := reports.OSFileSystem{}
223-
jsonFile, err := fs.Create(outputPath)
224-
if err != nil {
225-
return fmt.Errorf("error creating file: %w", err)
226-
}
227-
defer jsonFile.Close()
228-
229-
_, err = jsonFile.Write(data)
230-
if err != nil {
231-
return fmt.Errorf("error writing data to file: %w", err)
232-
}
233-
234-
return nil
235-
}

‎tools/flakeguard/cmd/generate_report.go

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ var GenerateReportCmd = &cobra.Command{
2525

2626
// Get flag values
2727
aggregatedResultsPath, _ := cmd.Flags().GetString("aggregated-results-path")
28-
summaryPath, _ := cmd.Flags().GetString("summary-path")
2928
outputDir, _ := cmd.Flags().GetString("output-path")
3029
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
3130
generatePRComment, _ := cmd.Flags().GetBool("generate-pr-comment")
@@ -65,28 +64,8 @@ var GenerateReportCmd = &cobra.Command{
6564
fmt.Println()
6665
log.Info().Msg("Successfully loaded aggregated test report")
6766

68-
// Load the summary data to check for failed tests
69-
var summaryData reports.SummaryData
70-
71-
if summaryPath == "" {
72-
log.Error().Msg("Summary path is required")
73-
os.Exit(ErrorExitCode)
74-
}
75-
76-
summaryFile, err := os.Open(summaryPath)
77-
if err != nil {
78-
log.Error().Err(err).Msg("Error opening summary JSON file")
79-
os.Exit(ErrorExitCode)
80-
}
81-
defer summaryFile.Close()
82-
83-
if err := json.NewDecoder(summaryFile).Decode(&summaryData); err != nil {
84-
log.Error().Err(err).Msg("Error decoding summary JSON file")
85-
os.Exit(ErrorExitCode)
86-
}
87-
8867
// Check if there are failed tests
89-
hasFailedTests := summaryData.FailedRuns > 0
68+
hasFailedTests := aggregatedReport.SummaryData.FailedRuns > 0
9069

9170
var artifactLink string
9271
if hasFailedTests {
@@ -185,7 +164,6 @@ var GenerateReportCmd = &cobra.Command{
185164

186165
func init() {
187166
GenerateReportCmd.Flags().StringP("aggregated-results-path", "i", "", "Path to the aggregated JSON report file (required)")
188-
GenerateReportCmd.Flags().StringP("summary-path", "s", "", "Path to the summary JSON file (required)")
189167
GenerateReportCmd.Flags().StringP("output-path", "o", "./report", "Path to output the generated report files")
190168
GenerateReportCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
191169
GenerateReportCmd.Flags().Bool("generate-pr-comment", false, "Set to true to generate PR comment markdown")
@@ -202,10 +180,6 @@ func init() {
202180
log.Error().Err(err).Msg("Error marking flag as required")
203181
os.Exit(ErrorExitCode)
204182
}
205-
if err := GenerateReportCmd.MarkFlagRequired("summary-path"); err != nil {
206-
log.Error().Err(err).Msg("Error marking flag as required")
207-
os.Exit(ErrorExitCode)
208-
}
209183
if err := GenerateReportCmd.MarkFlagRequired("github-repository"); err != nil {
210184
log.Error().Err(err).Msg("Error marking flag as required")
211185
os.Exit(ErrorExitCode)

‎tools/flakeguard/cmd/run.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,16 @@ var RunTestsCmd = &cobra.Command{
113113
return !tr.Skipped && tr.PassRatio < maxPassRatio
114114
})
115115

116-
fmt.Printf("\nFlakeguard Summary\n")
117-
reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false, false)
118116
if len(flakyTests) > 0 {
119117
log.Info().Int("count", len(flakyTests)).Str("pass ratio threshold", fmt.Sprintf("%.2f%%", maxPassRatio*100)).Msg("Found flaky tests")
118+
} else {
119+
log.Info().Msg("No flaky tests found")
120+
}
121+
122+
fmt.Printf("\nFlakeguard Summary\n")
123+
reports.RenderResults(os.Stdout, testReport, false, false)
124+
125+
if len(flakyTests) > 0 {
120126
// Exit with error code if there are flaky tests
121127
os.Exit(FlakyTestsExitCode)
122128
}

‎tools/flakeguard/reports/data.go

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ import (
55
"sort"
66
"strings"
77
"time"
8-
9-
"github.com/rs/zerolog/log"
108
)
119

1210
// TestReport reports on the parameters and results of one to many test runs
@@ -19,11 +17,13 @@ type TestReport struct {
1917
RepoURL string `json:"repo_url,omitempty"`
2018
GitHubWorkflowName string `json:"github_workflow_name,omitempty"`
2119
GitHubWorkflowRunURL string `json:"github_workflow_run_url,omitempty"`
22-
SummaryData SummaryData `json:"summary_data"`
20+
SummaryData *SummaryData `json:"summary_data"`
2321
RaceDetection bool `json:"race_detection"`
2422
ExcludedTests []string `json:"excluded_tests,omitempty"`
2523
SelectedTests []string `json:"selected_tests,omitempty"`
2624
Results []TestResult `json:"results,omitempty"`
25+
// MaxPassRatio is the maximum flakiness ratio allowed for a test to be considered not flaky
26+
MaxPassRatio float64 `json:"max_pass_ratio,omitempty"`
2727
}
2828

2929
// TestResult contains the results and outputs of a single test
@@ -81,8 +81,6 @@ type SummaryData struct {
8181
SkippedRuns int `json:"skipped_runs"`
8282
// PassPercent is the percentage of test runs that passed
8383
PassPercent string `json:"pass_percent"`
84-
// MaxPassRatio is the maximum flakiness ratio allowed for a test to be considered not flaky
85-
MaxPassRatio float64 `json:"max_pass_ratio,omitempty"`
8684
}
8785

8886
// SplunkType represents what type of data is being sent to Splunk, e.g. a report or a result.
@@ -131,18 +129,11 @@ type SplunkTestResultEvent struct {
131129

132130
// Data Processing Functions
133131

134-
// GenerateSummaryData generates a summary of the test results
135-
func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
136-
if maxPassRatio < 0 {
137-
log.Debug().Float64("ratio", maxPassRatio).Msg("maxPassRatio is negative, setting to 0")
138-
maxPassRatio = 0.0
139-
} else if maxPassRatio > 1 {
140-
log.Debug().Float64("ratio", maxPassRatio).Msg("maxPassRatio is greater than 1, setting to 1")
141-
maxPassRatio = 1.0
142-
}
132+
// GenerateSummaryData generates a summary of a report's test results
133+
func GenerateSummaryData(testReport *TestReport) {
143134
var runs, mostRuns, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
144135

145-
for _, result := range tests {
136+
for _, result := range testReport.Results {
146137
runs += result.Runs
147138
if result.Runs > mostRuns {
148139
mostRuns = result.Runs
@@ -162,7 +153,7 @@ func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
162153
} else if result.Race {
163154
racedTests++
164155
flakyTests++
165-
} else if !result.Skipped && result.Runs > 0 && result.PassRatio < maxPassRatio {
156+
} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
166157
flakyTests++
167158
}
168159
}
@@ -171,13 +162,13 @@ func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
171162
passRatio := passRatio(passes, runs)
172163

173164
// Calculate the raw flake ratio
174-
totalTests := len(tests)
165+
totalTests := len(testReport.Results)
175166
flakeRatio := flakeRatio(flakyTests, totalTests)
176167

177168
passRatioStr := formatRatio(passRatio)
178169
flakeTestRatioStr := formatRatio(flakeRatio)
179170

180-
return SummaryData{
171+
testReport.SummaryData = &SummaryData{
181172
UniqueTestsRun: totalTests,
182173
TestRunCount: mostRuns,
183174
PanickedTests: panickedTests,
@@ -189,9 +180,7 @@ func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
189180
PassedRuns: passes,
190181
FailedRuns: fails,
191182
SkippedRuns: skips,
192-
193-
PassPercent: passRatioStr,
194-
MaxPassRatio: maxPassRatio,
183+
PassPercent: passRatioStr,
195184
}
196185
}
197186

0 commit comments

Comments
 (0)