feat: start writing test report links in issue comments (#212)

* feat: start writing test report links in comments

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

internal/actions/test.go

@@ -1,19 +1,30 @@
 package actions
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
 	config "github.com/speakeasy-api/sdk-gen-config"
 	"github.com/speakeasy-api/sdk-generation-action/internal/cli"
 	"github.com/speakeasy-api/sdk-generation-action/internal/configuration"
 	"github.com/speakeasy-api/sdk-generation-action/internal/environment"
+	"github.com/speakeasy-api/sdk-generation-action/internal/git"
+	"github.com/speakeasy-api/sdk-generation-action/internal/telemetry"
 	"golang.org/x/exp/slices"
 )
 
-func Test() error {
+const testReportHeader = "SDK Tests Report"
+
+type TestReport struct {
+	Success bool
+	URL     string
+}
+
+func Test(ctx context.Context) error {
 	g, err := initAction()
 	if err != nil {
 		return err
@@ -30,26 +41,35 @@ func Test() error {
 
 	// This will only come in via workflow dispatch, we do accept 'all' as a special case
 	var testedTargets []string
-	if providedTargetName := environment.SpecifiedTarget(); providedTargetName != "" {
+	if providedTargetName := environment.SpecifiedTarget(); providedTargetName != "" && os.Getenv("GITHUB_EVENT_NAME") == "workflow_dispatch" {
 		testedTargets = append(testedTargets, providedTargetName)
 	}
 
+	var prNumber *int
+	targetLockIDs := make(map[string]string)
 	if len(testedTargets) == 0 {
 		// We look for all files modified in the PR or Branch to see what SDK targets have been modified
-		files, err := g.GetChangedFilesForPRorBranch()
+		files, number, err := g.GetChangedFilesForPRorBranch()
 		if err != nil {
 			fmt.Printf("Failed to get commited files: %s\n", err.Error())
 		}
 
+		prNumber = number
+
 		for _, file := range files {
 			if strings.Contains(file, "gen.yaml") || strings.Contains(file, "gen.lock") {
-				cfgDir := filepath.Dir(file)
-				_, err := config.Load(filepath.Dir(file))
+				configDir := filepath.Dir(filepath.Dir(file)) // gets out of .speakeasy
+				cfg, err := config.Load(filepath.Join(environment.GetWorkspace(), "repo", configDir))
 				if err != nil {
 					return fmt.Errorf("failed to load config: %w", err)
 				}
 
-				outDir, err := filepath.Abs(filepath.Dir(cfgDir))
+				var genLockID string
+				if cfg.LockFile != nil {
+					genLockID = cfg.LockFile.ID
+				}
+
+				outDir, err := filepath.Abs(configDir)
 				if err != nil {
 					return err
 				}
@@ -65,6 +85,7 @@ func Test() error {
 					// If there are multiple SDKs in a workflow we ensure output path is unique
 					if targetOutput == outDir && !slices.Contains(testedTargets, name) {
 						testedTargets = append(testedTargets, name)
+						targetLockIDs[name] = genLockID
 					}
 				}
 			}
@@ -77,12 +98,36 @@ func Test() error {
 
 	// we will pretty much never have a test action for multiple targets
 	// but if a customer manually setup their triggers in this way, we will run test sequentially for clear output
+
+	testReports := make(map[string]TestReport)
 	var errs []error
 	for _, target := range testedTargets {
-		// TODO: Once we have stable test reports we will probably want to use GH API to leave a PR comment/clean up old comments
-		if err := cli.Test(target); err != nil {
+		err := cli.Test(target)
+		if err != nil {
 			errs = append(errs, err)
 		}
+
+		testReportURL := ""
+		if genLockID, ok := targetLockIDs[target]; ok && genLockID != "" {
+			testReportURL = formatTestReportURL(ctx, genLockID)
+		} else {
+			fmt.Println(fmt.Sprintf("No gen.lock ID found for target %s", target))
+		}
+
+		if testReportURL == "" {
+			fmt.Println(fmt.Sprintf("No test report URL could be formed for target %s", target))
+		} else {
+			testReports[target] = TestReport{
+				Success: err == nil,
+				URL:     testReportURL,
+			}
+		}
+	}
+
+	if len(testReports) > 0 {
+		if err := writeTestReportComment(g, prNumber, testReports); err != nil {
+			fmt.Println(fmt.Sprintf("Failed to write test report comment: %s\n", err.Error()))
+		}
 	}
 
 	if len(errs) > 0 {
@@ -91,3 +136,64 @@ func Test() error {
 
 	return nil
 }
+
+func formatTestReportURL(ctx context.Context, genLockID string) string {
+	executionID := os.Getenv(telemetry.ExecutionKeyEnvironmentVariable)
+	if executionID == "" {
+		return ""
+	}
+
+	if ctx.Value(telemetry.OrgSlugKey) == nil {
+		return ""
+	}
+	orgSlug, ok := ctx.Value(telemetry.OrgSlugKey).(string)
+	if !ok {
+		return ""
+	}
+
+	if ctx.Value(telemetry.WorkspaceSlugKey) == nil {
+		return ""
+	}
+	workspaceSlug, ok := ctx.Value(telemetry.WorkspaceSlugKey).(string)
+	if !ok {
+		return ""
+	}
+
+	return fmt.Sprintf("https://app.speakeasy.com/org/%s/%s/targets/%s/tests/%s", orgSlug, workspaceSlug, genLockID, executionID)
+}
+
+func writeTestReportComment(g *git.Git, prNumber *int, testReports map[string]TestReport) error {
+	if prNumber == nil {
+		return fmt.Errorf("PR number is nil, cannot post comment")
+	}
+
+	currentPRComments, _ := g.ListIssueComments(*prNumber)
+	for _, comment := range currentPRComments {
+		commentBody := comment.GetBody()
+		if strings.Contains(commentBody, testReportHeader) {
+			if err := g.DeleteIssueComment(comment.GetID()); err != nil {
+				fmt.Println(fmt.Sprintf("Failed to delete existing test report comment: %s\n", err.Error()))
+			}
+		}
+	}
+
+	titleComment := fmt.Sprintf("## **%s**\n\n", testReportHeader)
+
+	tableHeader := "| Target | Status | Report |\n|--------|--------|--------|\n"
+
+	var tableRows strings.Builder
+	for target, report := range testReports {
+		statusEmoji := "✅"
+		if !report.Success {
+			statusEmoji = "❌"
+		}
+		tableRows.WriteString(fmt.Sprintf("| %s | <p align='center'>%s</p> | [view report](%s) |\n", target, statusEmoji, report.URL))
+	}
+
+	// Combine everything
+	body := titleComment + tableHeader + tableRows.String()
+
+	err := g.WriteIssueComment(*prNumber, body)
+
+	return err
+}

internal/git/git.go

@@ -875,6 +875,24 @@ func (g *Git) WritePRBody(prNumber int, body string) error {
 	return nil
 }
 
+func (g *Git) ListIssueComments(prNumber int) ([]*github.IssueComment, error) {
+	comments, _, err := g.client.Issues.ListComments(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), prNumber, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get PR comments: %w", err)
+	}
+
+	return comments, nil
+}
+
+func (g *Git) DeleteIssueComment(commentID int64) error {
+	_, err := g.client.Issues.DeleteComment(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), commentID)
+	if err != nil {
+		return fmt.Errorf("failed to delete issue comment: %w", err)
+	}
+
+	return nil
+}
+
 func (g *Git) WritePRComment(prNumber int, fileName, body string, line int) error {
 	pr, _, err := g.client.PullRequests.Get(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), prNumber)
 	if err != nil {
@@ -894,6 +912,19 @@ func (g *Git) WritePRComment(prNumber int, fileName, body string, line int) erro
 	return nil
 }
 
+func (g *Git) WriteIssueComment(prNumber int, body string) error {
+	comment := &github.IssueComment{
+		Body: github.String(sanitizeExplanations(body)),
+	}
+
+	_, _, err := g.client.Issues.CreateComment(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), prNumber, comment)
+	if err != nil {
+		return fmt.Errorf("failed to create issue comment: %w", err)
+	}
+
+	return nil
+}
+
 func sanitizeExplanations(str string) string {
 	// Remove ANSI sequences
 	ansiEscape := regexp.MustCompile(`\x1b[^m]*m`)
@@ -1045,16 +1076,16 @@ func getDownloadLinkFromReleases(releases []*github.RepositoryRelease, version s
 	return defaultDownloadUrl, defaultTagName
 }
 
-func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
+func (g *Git) GetChangedFilesForPRorBranch() ([]string, *int, error) {
 	ctx := context.Background()
 	eventPath := os.Getenv("GITHUB_EVENT_PATH")
 	if eventPath == "" {
-		return nil, fmt.Errorf("no workflow event payload path")
+		return nil, nil, fmt.Errorf("no workflow event payload path")
 	}
 
 	data, err := os.ReadFile(eventPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read workflow event payload: %w", err)
+		return nil, nil, fmt.Errorf("failed to read workflow event payload: %w", err)
 	}
 
 	var payload struct {
@@ -1065,16 +1096,17 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 	}
 
 	if err := json.Unmarshal(data, &payload); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal workflow event payload: %w", err)
+		return nil, nil, fmt.Errorf("failed to unmarshal workflow event payload: %w", err)
 	}
 
 	prNumber := payload.Number
 	// This occurs if we come from a non-PR event trigger
 	if payload.Number == 0 {
 		ref := strings.TrimPrefix(environment.GetRef(), "refs/heads/")
 		if ref == "main" || ref == "master" {
+			files, err := g.GetCommitedFiles()
 			// We just need to get the commit diff since we are not in a separate branch of PR
-			return g.GetCommitedFiles()
+			return files, nil, err
 		}
 
 		opts := &github.PullRequestListOptions{
@@ -1083,6 +1115,7 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 		}
 
 		if prs, _, _ := g.client.PullRequests.List(ctx, os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), opts); len(prs) > 0 {
+			prNumber = prs[0].GetNumber()
 			os.Setenv("GH_PULL_REQUEST", prs[0].GetURL())
 		}
 
@@ -1095,13 +1128,13 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 		// Get the feature branch reference
 		branchRef, err := g.repo.Reference(plumbing.ReferenceName(environment.GetRef()), true)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get feature branch reference: %w", err)
+			return nil, nil, fmt.Errorf("failed to get feature branch reference: %w", err)
 		}
 
 		// Get the latest commit on the feature branch
 		latestCommit, err := g.repo.CommitObject(branchRef.Hash())
 		if err != nil {
-			return nil, fmt.Errorf("failed to get latest commit of feature branch: %w", err)
+			return nil, nil, fmt.Errorf("failed to get latest commit of feature branch: %w", err)
 		}
 
 		var files []string
@@ -1118,7 +1151,7 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 				opt,
 			)
 			if err != nil {
-				return nil, fmt.Errorf("failed to compare commits via GitHub API: %w", err)
+				return nil, nil, fmt.Errorf("failed to compare commits via GitHub API: %w", err)
 			}
 
 			// Collect filenames from this page
@@ -1136,7 +1169,7 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 		}
 
 		logging.Info("Found %d files", len(files))
-		return files, nil
+		return files, &prNumber, nil
 
 	} else {
 		prURL := fmt.Sprintf("https://github.com/%s/%s/pull/%d", os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), prNumber)
@@ -1148,7 +1181,7 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 		for {
 			files, resp, err := g.client.PullRequests.ListFiles(ctx, os.Getenv("GITHUB_REPOSITORY_OWNER"), GetRepo(), prNumber, opts)
 			if err != nil {
-				return nil, fmt.Errorf("failed to get changed files: %w", err)
+				return nil, nil, fmt.Errorf("failed to get changed files: %w", err)
 			}
 
 			for _, file := range files {
@@ -1163,7 +1196,7 @@ func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
 
 		logging.Info("Found %d files", len(allFiles))
 
-		return allFiles, nil
+		return allFiles, &prNumber, nil
 	}
 }
 

internal/telemetry/telemetry.go

@@ -24,6 +24,8 @@ const ExecutionKeyEnvironmentVariable = "SPEAKEASY_EXECUTION_ID"
 const SpeakeasySDKKey ContextKey = "speakeasy.SDK"
 const WorkspaceIDKey ContextKey = "speakeasy.workspaceID"
 const AccountTypeKey ContextKey = "speakeasy.accountType"
+const WorkspaceSlugKey ContextKey = "speakeasy.workspaceSlug"
+const OrgSlugKey ContextKey = "speakeasy.orgSlug"
 
 // a random UUID. Change this to fan-out executions with the same gh run id.
 const speakeasyGithubActionNamespace = "360D564A-5583-4EF6-BC2B-99530BF036CC"
@@ -46,6 +48,8 @@ func NewContextWithSDK(ctx context.Context, apiKey string) (context.Context, *sp
 	ctx = context.WithValue(ctx, SpeakeasySDKKey, sdkWithWorkspace)
 	ctx = context.WithValue(ctx, WorkspaceIDKey, validated.APIKeyDetails.WorkspaceID)
 	ctx = context.WithValue(ctx, AccountTypeKey, validated.APIKeyDetails.AccountTypeV2)
+	ctx = context.WithValue(ctx, WorkspaceSlugKey, validated.APIKeyDetails.WorkspaceSlug)
+	ctx = context.WithValue(ctx, OrgSlugKey, validated.APIKeyDetails.OrgSlug)
 	return ctx, sdkWithWorkspace, validated.APIKeyDetails.WorkspaceID, err
 }
 

main.go

@@ -55,7 +55,7 @@ func main() {
 			case environment.ActionTag:
 				return actions.Tag()
 			case environment.ActionTest:
-				return actions.Test()
+				return actions.Test(ctx)
 			default:
 				return fmt.Errorf("unknown action: %s", environment.GetAction())
 			}