Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds an execution Error Code to Flakeguard #1617

Merged
merged 2 commits into from
Feb 5, 2025
Merged

Adds an execution Error Code to Flakeguard #1617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ae9d59
Adds an execution error code to Flakeguard
kalverra Feb 5, 2025
540011f
Merge branch 'main' of github.com:smartcontractkit/chainlink-testing-…
kalverra Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 25 additions & 16 deletions tools/flakeguard/cmd/aggregate_results.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"

Expand All @@ -15,7 +16,7 @@ import (
var AggregateResultsCmd = &cobra.Command{
Use: "aggregate-results",
Short: "Aggregate test results into a single JSON report",
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
fs := reports.OSFileSystem{}

// Get flag values
Expand All @@ -37,7 +38,8 @@ var AggregateResultsCmd = &cobra.Command{

// Ensure the output directory exists
if err := fs.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("error creating output directory: %w", err)
log.Error().Err(err).Str("path", outputDir).Msg("Error creating output directory")
os.Exit(ErrorExitCode)
}

// Start spinner for loading test reports
Expand All @@ -59,16 +61,11 @@ var AggregateResultsCmd = &cobra.Command{
if err != nil {
s.Stop()
fmt.Println()
return fmt.Errorf("error loading test reports: %w", err)
log.Error().Err(err).Msg("Error aggregating test reports")
os.Exit(ErrorExitCode)
}
s.Stop()
fmt.Println()

if err != nil {
s.Stop()
return fmt.Errorf("error aggregating test reports: %w", err)
}
s.Stop()
log.Debug().Msg("Successfully loaded and aggregated test reports")

// Start spinner for mapping test results to paths
Expand All @@ -80,9 +77,12 @@ var AggregateResultsCmd = &cobra.Command{
err = reports.MapTestResultsToPaths(aggregatedReport, repoPath)
if err != nil {
s.Stop()
return fmt.Errorf("error mapping test results to paths: %w", err)
fmt.Println()
log.Error().Err(err).Msg("Error mapping test results to paths")
os.Exit(ErrorExitCode)
}
s.Stop()
fmt.Println()
log.Debug().Msg("Successfully mapped paths to test results")

// Map test results to code owners if codeOwnersPath is provided
Expand All @@ -94,16 +94,20 @@ var AggregateResultsCmd = &cobra.Command{
err = reports.MapTestResultsToOwners(aggregatedReport, codeOwnersPath)
if err != nil {
s.Stop()
return fmt.Errorf("error mapping test results to code owners: %w", err)
fmt.Println()
log.Error().Err(err).Msg("Error mapping test results to code owners")
os.Exit(ErrorExitCode)
}
s.Stop()
fmt.Println()
log.Debug().Msg("Successfully mapped code owners to test results")
}

failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
return !tr.Skipped && tr.PassRatio < maxPassRatio
})
s.Stop()
fmt.Println()

// Check if there are any failed tests
if len(failedTests) > 0 {
Expand All @@ -125,7 +129,8 @@ var AggregateResultsCmd = &cobra.Command{
// Save the failed tests report with logs
failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
return fmt.Errorf("error saving failed tests report with logs: %w", err)
log.Error().Err(err).Msg("Error saving failed tests report with logs")
os.Exit(ErrorExitCode)
}
log.Debug().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")

Expand All @@ -139,7 +144,8 @@ var AggregateResultsCmd = &cobra.Command{
// Save the failed tests report without logs
failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-results.json")
if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportWithLogs); err != nil {
return fmt.Errorf("error saving failed tests report without logs: %w", err)
log.Error().Err(err).Msg("Error saving failed tests report without logs")
os.Exit(ErrorExitCode)
}
log.Debug().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
} else {
Expand All @@ -156,7 +162,8 @@ var AggregateResultsCmd = &cobra.Command{
// Save the aggregated report to the output directory
aggregatedReportPath := filepath.Join(outputDir, "all-test-results.json")
if err := reports.SaveReport(fs, aggregatedReportPath, *aggregatedReport); err != nil {
return fmt.Errorf("error saving aggregated test report: %w", err)
log.Error().Err(err).Msg("Error saving aggregated test report")
os.Exit(ErrorExitCode)
}
log.Debug().Str("path", aggregatedReportPath).Msg("Aggregated test report saved")

Expand All @@ -171,14 +178,16 @@ var AggregateResultsCmd = &cobra.Command{
err = generateAllTestsSummaryJSON(aggregatedReport, summaryFilePath, maxPassRatio)
if err != nil {
s.Stop()
return fmt.Errorf("error generating summary json: %w", err)
fmt.Println()
log.Error().Err(err).Msg("Error generating summary json")
os.Exit(ErrorExitCode)
}
s.Stop()
fmt.Println()
log.Debug().Str("path", summaryFilePath).Msg("Summary generated")
}

log.Info().Str("summary", summaryFilePath).Str("report", aggregatedReportPath).Msg("Aggregation complete")
return nil
},
}

Expand Down
11 changes: 6 additions & 5 deletions tools/flakeguard/cmd/check_test_owners.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"os"
"path/filepath"

"github.com/rs/zerolog/log"
Expand All @@ -18,19 +19,21 @@ var (
var CheckTestOwnersCmd = &cobra.Command{
Use: "check-test-owners",
Short: "Check which tests in the project do not have code owners",
RunE: func(cmd *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
projectPath, _ := cmd.Flags().GetString("project-path")

// Scan project for test functions
testFileMap, err := reports.ScanTestFiles(projectPath)
if err != nil {
log.Fatal().Err(err).Msg("Error scanning test files")
log.Error().Err(err).Msg("Error scanning test files")
os.Exit(ErrorExitCode)
}

// Parse CODEOWNERS file
codeOwnerPatterns, err := codeowners.Parse(codeownersPath)
if err != nil {
log.Fatal().Err(err).Msg("Error parsing CODEOWNERS file")
log.Error().Err(err).Msg("Error parsing CODEOWNERS file")
os.Exit(ErrorExitCode)
}

// Check for tests without code owners
Expand Down Expand Up @@ -73,8 +76,6 @@ var CheckTestOwnersCmd = &cobra.Command{
log.Debug().Msg("All Test functions have code owners!")
}
}

return nil
},
}

Expand Down
31 changes: 21 additions & 10 deletions tools/flakeguard/cmd/find.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"encoding/json"
"fmt"
"os"

"github.com/rs/zerolog/log"
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/git"
Expand Down Expand Up @@ -32,7 +33,8 @@ var FindTestsCmd = &cobra.Command{
if findByTestFilesDiff {
changedTestFiles, err := git.FindChangedFiles(projectPath, baseRef, "grep '_test\\.go$'")
if err != nil {
log.Fatal().Err(err).Msg("Error finding changed test files")
log.Error().Err(err).Msg("Error finding changed test files")
os.Exit(ErrorExitCode)
}
if onlyShowChangedTestFiles {
outputResults(changedTestFiles, jsonOutput)
Expand All @@ -43,7 +45,8 @@ var FindTestsCmd = &cobra.Command{
}
changedTestPkgs, err = golang.GetFilePackages(changedTestFiles)
if err != nil {
log.Fatal().Err(err).Msg("Error getting package names for test files")
log.Error().Err(err).Msg("Error getting package names for test files")
os.Exit(ErrorExitCode)
}
}

Expand Down Expand Up @@ -85,40 +88,47 @@ func init() {
FindTestsCmd.Flags().Bool("only-show-changed-test-files", false, "Only show the changed test files and exit")

if err := FindTestsCmd.MarkFlagRequired("base-ref"); err != nil {
log.Fatal().Err(err).Msg("Error marking base-ref as required")
log.Error().Err(err).Msg("Error marking base-ref as required")
os.Exit(ErrorExitCode)
}
}

func findAffectedPackages(baseRef, projectPath string, excludes []string, levels int) []string {
goList, err := golang.GoList()
if err != nil {
log.Fatal().Err(err).Msg("Error getting go list")
log.Error().Err(err).Msg("Error getting go list")
os.Exit(ErrorExitCode)
}
gitDiff, err := git.Diff(baseRef)
if err != nil {
log.Fatal().Err(err).Msg("Error getting the git diff")
log.Error().Err(err).Msg("Error getting the git diff")
os.Exit(ErrorExitCode)
}
gitModDiff, err := git.ModDiff(baseRef, projectPath)
if err != nil {
log.Fatal().Err(err).Msg("Error getting the git mod diff")
log.Error().Err(err).Msg("Error getting the git mod diff")
os.Exit(ErrorExitCode)
}

packages, err := golang.ParsePackages(goList.Stdout)
if err != nil {
log.Fatal().Err(err).Msg("Error parsing packages")
log.Error().Err(err).Msg("Error parsing packages")
os.Exit(ErrorExitCode)
}

fileMap := golang.GetGoFileMap(packages, true)

var changedPackages []string
changedPackages, err = git.GetChangedGoPackagesFromDiff(gitDiff.Stdout, projectPath, excludes, fileMap)
if err != nil {
log.Fatal().Err(err).Msg("Error getting changed packages")
log.Error().Err(err).Msg("Error getting changed packages")
os.Exit(ErrorExitCode)
}

changedModPackages, err := git.GetGoModChangesFromDiff(gitModDiff.Stdout)
if err != nil {
log.Fatal().Err(err).Msg("Error getting go.mod changes")
log.Error().Err(err).Msg("Error getting go.mod changes")
os.Exit(ErrorExitCode)
}

depMap := golang.GetGoDepMap(packages)
Expand Down Expand Up @@ -156,7 +166,8 @@ func outputResults(packages []string, jsonOutput bool) {
if jsonOutput {
data, err := json.Marshal(packages)
if err != nil {
log.Fatal().Err(err).Msg("Error marshaling test packages to JSON")
log.Error().Err(err).Msg("Error marshaling test packages to JSON")
os.Exit(ErrorExitCode)
}
log.Debug().Str("output", string(data)).Msg("JSON")
} else {
Expand Down
Loading
Loading