Merge pull request #48 from DerekTBrown/fetch_metrics_for_orgs

feat: fetch all repositories

Author: moutonjeremy

README.md

@@ -25,7 +25,7 @@ Authentication can either via a Github Token or the Github App Authentication 3
 | Github App Private Key | app_private_key, gpk | GITHUB_APP_PRIVATE_KEY | - | Github App Authentication Private Key |
 | Github Refresh | github_refresh, gr | GITHUB_REFRESH | 30 | Refresh time Github Actions status in sec |
 | Github Organizations | github_orgas, go | GITHUB_ORGAS | - | List all organizations you want get informations. Format \<orga1>,\<orga2>,\<orga3> (like test1,test2) |
-| Github Repos | github_repos, grs | GITHUB_REPOS | - | List all repositories you want get informations. Format \<orga>/\<repo>,\<orga>/\<repo2>,\<orga>/\<repo3> (like test/test) |
+| Github Repos | github_repos, grs | GITHUB_REPOS | - | [Optional] List all repositories you want get informations. Format \<orga>/\<repo>,\<orga>/\<repo2>,\<orga>/\<repo3> (like test/test). Defaults to all repositories owned by the organizations. |
 | Exporter port | port, p | PORT | 9999 | Exporter port |
 | Github Api URL | github_api_url, url | GITHUB_API_URL | api.github.com | Github API URL (primarily for Github Enterprise usage) |
 | Github Enterprise Name | enterprise_name | ENTERPRISE_NAME | "" | Enterprise name. Needed for enterprise endpoints (/enterprises/{ENTERPRISE_NAME}/*). Currently used to get Enterprise level tunners status |

go.mod

@@ -5,12 +5,11 @@ go 1.16
 require (
 	github.com/andybalholm/brotli v1.0.3 // indirect
 	github.com/bradleyfalzon/ghinstallation v1.1.1
+	github.com/die-net/lrucache v0.0.0-20220628165024-20a71bc65bf1 // indirect
 	github.com/fasthttp/router v1.3.9
-	github.com/go-kit/kit v0.10.0 // indirect
-	github.com/google/go-github v17.0.0+incompatible // indirect
-	github.com/google/go-github/v38 v38.1.0
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-github/v45 v45.2.0
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/prometheus/client_golang v1.11.0
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/valyala/fasthttp v1.22.0

go.sum

@@ -12,12 +12,16 @@ var (
 		Refresh           int64
 		Repositories      cli.StringSlice
 		Organizations     cli.StringSlice
-		APIURL            string			
+		APIURL            string
+		CacheSizeBytes    int64
 	}
-	Port  int
-	Debug bool
-	EnterpriseName    string
-	WorkflowFields    string
+	Metrics struct {
+		FetchWorkflowRunUsage bool
+	}
+	Port           int
+	Debug          bool
+	EnterpriseName string
+	WorkflowFields string
 )
 
 // InitConfiguration - set configuration from env vars or command parameters
@@ -109,5 +113,19 @@ func InitConfiguration() []cli.Flag {
 			Value:       "repo,id,node_id,head_branch,head_sha,run_number,workflow_id,workflow,event,status",
 			Destination: &WorkflowFields,
 		},
+		&cli.BoolFlag{
+			Name:        "fetch_workflow_run_usage",
+			EnvVars:     []string{"FETCH_WORKFLOW_RUN_USAGE"},
+			Usage:       "When true, will perform an API call per workflow run to fetch the workflow usage",
+			Value:       true,
+			Destination: &Metrics.FetchWorkflowRunUsage,
+		},
+		&cli.Int64Flag{
+			Name:        "github_cache_size_bytes",
+			EnvVars:     []string{"GITHUB_CACHE_SIZE_BYTES"},
+			Value:       100 * 1024 * 1024,
+			Usage:       "Size of Github HTTP cache in bytes",
+			Destination: &Github.CacheSizeBytes,
+		},
 	}
 }

pkg/config/config.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/go-github/github"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -24,17 +25,26 @@ var (
 // getBillableFromGithub - return billable informations for MACOS, WINDOWS and UBUNTU runners.
 func getBillableFromGithub() {
 	for {
-		for _, repo := range config.Github.Repositories.Value() {
+		for _, repo := range repositories {
 			for k, v := range workflows[repo] {
 				r := strings.Split(repo, "/")
-				resp, _, err := client.Actions.GetWorkflowUsageByID(context.Background(), r[0], r[1], k)
-				if err != nil {
-					log.Printf("GetWorkflowUsageByID error for %s: %s", repo, err.Error())
-				} else {
+
+				for {
+					resp, _, err := client.Actions.GetWorkflowUsageByID(context.Background(), r[0], r[1], k)
+					if rl_err, ok := err.(*github.RateLimitError); ok {
+						log.Printf("GetWorkflowUsageByID ratelimited. Pausing until %s", rl_err.Rate.Reset.Time.String())
+						time.Sleep(time.Until(rl_err.Rate.Reset.Time))
+						continue
+					} else if err != nil {
+						log.Printf("GetWorkflowUsageByID error for %s: %s", repo, err.Error())
+						break
+					}
 					workflowBillGauge.WithLabelValues(repo, strconv.FormatInt(*v.ID, 10), *v.NodeID, *v.Name, *v.State, "MACOS").Set(float64(resp.GetBillable().MacOS.GetTotalMS()) / 1000)
 					workflowBillGauge.WithLabelValues(repo, strconv.FormatInt(*v.ID, 10), *v.NodeID, *v.Name, *v.State, "WINDOWS").Set(float64(resp.GetBillable().Windows.GetTotalMS()) / 1000)
 					workflowBillGauge.WithLabelValues(repo, strconv.FormatInt(*v.ID, 10), *v.NodeID, *v.Name, *v.State, "UBUNTU").Set(float64(resp.GetBillable().Ubuntu.GetTotalMS()) / 1000)
+					break
 				}
+
 			}
 		}
 

pkg/metrics/get_billable_from_github.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/google/go-github/v45/github"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -20,22 +21,46 @@ var (
 	)
 )
 
+func getAllEnterpriseRunners() []*github.Runner {
+	var runners []*github.Runner
+	opt := &github.ListOptions{PerPage: 200}
+
+	for {
+		resp, rr, err := client.Enterprise.ListRunners(context.Background(), config.EnterpriseName, nil)
+		if rl_err, ok := err.(*github.RateLimitError); ok {
+			log.Printf("ListRunners ratelimited. Pausing until %s", rl_err.Rate.Reset.Time.String())
+			time.Sleep(time.Until(rl_err.Rate.Reset.Time))
+			continue
+		} else if err != nil {
+			log.Printf("ListRunners error for enterprise %s: %s", config.EnterpriseName, err.Error())
+			return nil
+		}
+
+		runners = append(runners, resp.Runners...)
+		if rr.NextPage == 0 {
+			break
+		}
+		opt.Page = rr.NextPage
+	}
+
+	return runners
+}
+
 func getRunnersEnterpriseFromGithub() {
-	if config.EnterpriseName != "" {
-		for {
-			runners, _, err := client.Enterprise.ListRunners(context.Background(), config.EnterpriseName, nil)
-			if err != nil {
-				log.Printf("Enterprise.ListRunners error: %s", err.Error())
-			} else {
-				for _, runner := range runners.Runners {
-					var integerStatus float64
-					if integerStatus = 0; runner.GetStatus() == "online" {
-						integerStatus = 1
-					}
-					runnersEnterpriseGauge.WithLabelValues(*runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(integerStatus)
-				}
+	if config.EnterpriseName == "" {
+		return
+	}
+	for {
+		runners := getAllEnterpriseRunners()
+
+		for _, runner := range runners {
+			var integerStatus float64
+			if integerStatus = 0; runner.GetStatus() == "online" {
+				integerStatus = 1
 			}
+			runnersEnterpriseGauge.WithLabelValues(*runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10)).Set(integerStatus)
+		}
+
 		time.Sleep(time.Duration(config.Github.Refresh) * time.Second)
 	}
-	}
 }

pkg/metrics/get_runners_enterprise_from_github.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/go-github/v45/github"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -21,21 +22,43 @@ var (
 	)
 )
 
+func getAllRepoRunners(owner string, repo string) []*github.Runner {
+	var runners []*github.Runner
+	opt := &github.ListOptions{PerPage: 200}
+
+	for {
+		resp, rr, err := client.Actions.ListRunners(context.Background(), owner, repo, opt)
+		if rl_err, ok := err.(*github.RateLimitError); ok {
+			log.Printf("ListRunners ratelimited. Pausing until %s", rl_err.Rate.Reset.Time.String())
+			time.Sleep(time.Until(rl_err.Rate.Reset.Time))
+			continue
+		} else if err != nil {
+			log.Printf("ListRunners error for repo %s: %s", repo, err.Error())
+			return nil
+		}
+
+		runners = append(runners, resp.Runners...)
+		if rr.NextPage == 0 {
+			break
+		}
+		opt.Page = rr.NextPage
+	}
+
+	return runners
+}
+
 // getRunnersFromGithub - return information about runners and their status for a specific repo
 func getRunnersFromGithub() {
 	for {
-		for _, repo := range config.Github.Repositories.Value() {
+		for _, repo := range repositories {
 			r := strings.Split(repo, "/")
-			resp, _, err := client.Actions.ListRunners(context.Background(), r[0], r[1], nil)
-			if err != nil {
-				log.Printf("ListRunners error for %s: %s", repo, err.Error())
-			} else {
-				for _, runner := range resp.Runners {
-					if runner.GetStatus() == "online" {
-						runnersGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(1)
-					} else {
-						runnersGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(0)
-					}
+
+			runners := getAllRepoRunners(r[0], r[1])
+			for _, runner := range runners {
+				if runner.GetStatus() == "online" {
+					runnersGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(1)
+				} else {
+					runnersGauge.WithLabelValues(repo, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(0)
 				}
 			}
 		}

pkg/metrics/get_runners_from_github.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/google/go-github/v38/github"
+	"github.com/google/go-github/v45/github"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -21,27 +21,40 @@ var (
 	)
 )
 
+func getAllOrgRunners(orga string) []*github.Runner {
+	var runners []*github.Runner
+	opt := &github.ListOptions{PerPage: 200}
+
+	for {
+		resp, rr, err := client.Actions.ListOrganizationRunners(context.Background(), orga, opt)
+		if rl_err, ok := err.(*github.RateLimitError); ok {
+			log.Printf("ListOrganizationRunners ratelimited. Pausing until %s", rl_err.Rate.Reset.Time.String())
+			time.Sleep(time.Until(rl_err.Rate.Reset.Time))
+			continue
+		} else if err != nil {
+			log.Printf("ListOrganizationRunners error for org %s: %s", orga, err.Error())
+			return runners
+		}
+
+		runners = append(runners, resp.Runners...)
+		if rr.NextPage == 0 {
+			break
+		}
+		opt.Page = rr.NextPage
+	}
+	return runners
+}
+
 // getRunnersOrganizationFromGithub - return information about runners and their status for an organization
 func getRunnersOrganizationFromGithub() {
 	for {
 		for _, orga := range config.Github.Organizations.Value() {
-			opt := &github.ListOptions{PerPage: 10}
-			for {
-				resp, rr, err := client.Actions.ListOrganizationRunners(context.Background(), orga, opt)
-				if err != nil {
-					log.Printf("ListOrganizationRunners error for %s: %s", orga, err.Error())
+			runners := getAllOrgRunners(orga)
+			for _, runner := range runners {
+				if runner.GetStatus() == "online" {
+					runnersOrganizationGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(1)
 				} else {
-					for _, runner := range resp.Runners {
-						if runner.GetStatus() == "online" {
-							runnersOrganizationGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(1)
-						} else {
-							runnersOrganizationGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(0)
-						}
-					}
-					if rr.NextPage == 0 {
-						break
-					}
-					opt.Page = rr.NextPage
+					runnersOrganizationGauge.WithLabelValues(orga, *runner.OS, *runner.Name, strconv.FormatInt(runner.GetID(), 10), strconv.FormatBool(runner.GetBusy())).Set(0)
 				}
 			}
 		}