diff --git a/pkg/job/scraper.go b/pkg/job/scraper.go deleted file mode 100644 index 731649796..000000000 --- a/pkg/job/scraper.go +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2024 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package job - -import ( - "context" - "fmt" - "log/slog" - "sync" - - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/job/cloudwatchrunner" - - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/clients/account" - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/model" -) - -type Scraper struct { - jobsCfg model.JobsConfig - logger *slog.Logger - runnerFactory runnerFactory -} - -type runnerFactory interface { - GetAccountClient(region string, role model.Role) account.Client - NewResourceMetadataRunner(logger *slog.Logger, region string, role model.Role) ResourceMetadataRunner - NewCloudWatchRunner(logger *slog.Logger, region string, role model.Role, job cloudwatchrunner.Job) CloudwatchRunner -} - -type ResourceMetadataRunner interface { - Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) -} - -type CloudwatchRunner interface { - Run(ctx context.Context) ([]*model.CloudwatchData, error) -} - -func NewScraper(logger *slog.Logger, - jobsCfg model.JobsConfig, - runnerFactory runnerFactory, -) *Scraper { - return &Scraper{ - runnerFactory: runnerFactory, - logger: logger, - jobsCfg: jobsCfg, - } -} - -type ErrorType string - -var ( - AccountErr ErrorType = "Account for job was not found" - ResourceMetadataErr ErrorType = "Failed to run resource metadata for job" - CloudWatchCollectionErr ErrorType = "Failed to gather cloudwatch metrics for job" -) - -type Account struct { - ID string - Alias string -} - -func (s Scraper) Scrape(ctx context.Context) ([]model.TaggedResourceResult, []model.CloudwatchMetricResult, []Error) { - // Setup so we only do one GetAccount call per region + role combo when running jobs - roleRegionToAccount := map[model.Role]map[string]func() (Account, error){} - jobConfigVisitor(s.jobsCfg, func(_ any, role model.Role, region string) { - if _, exists := roleRegionToAccount[role]; !exists { - roleRegionToAccount[role] = map[string]func() (Account, error){} - } - roleRegionToAccount[role][region] = sync.OnceValues[Account, error](func() (Account, error) { - client := s.runnerFactory.GetAccountClient(region, role) - accountID, err := client.GetAccount(ctx) - if err != nil { - return Account{}, fmt.Errorf("failed to get Account: %w", err) - } - a := Account{ - ID: accountID, - } - accountAlias, err := client.GetAccountAlias(ctx) - if err != nil { - s.logger.Warn("Failed to get optional account alias from account", "err", err, "account_id", accountID) - } else { - a.Alias = accountAlias - } - return a, nil - }) - }) - - var wg sync.WaitGroup - mux := &sync.Mutex{} - jobErrors := make([]Error, 0) - metricResults := make([]model.CloudwatchMetricResult, 0) - resourceResults := make([]model.TaggedResourceResult, 0) - s.logger.Debug("Starting job runs") - - jobConfigVisitor(s.jobsCfg, func(job any, role model.Role, region string) { - wg.Add(1) - go func() { - defer wg.Done() - - var namespace string - jobAction(s.logger, job, func(job model.DiscoveryJob) { - namespace = job.Namespace - }, func(job model.CustomNamespaceJob) { - namespace = job.Namespace - }) - jobContext := JobContext{ - Namespace: namespace, - Region: region, - RoleARN: role.RoleArn, - } - jobLogger := s.logger.With("namespace", jobContext.Namespace, "region", jobContext.Region, "arn", jobContext.RoleARN) - - account, err := roleRegionToAccount[role][region]() - if err != nil { - jobError := NewError(jobContext, AccountErr, err) - mux.Lock() - jobErrors = append(jobErrors, jobError) - mux.Unlock() - return - } - jobContext.Account = account - jobLogger = jobLogger.With("account_id", jobContext.Account.ID) - - var jobToRun cloudwatchrunner.Job - jobAction(jobLogger, job, - func(job model.DiscoveryJob) { - jobLogger.Debug("Starting resource discovery") - rmRunner := s.runnerFactory.NewResourceMetadataRunner(jobLogger, region, role) - resources, err := rmRunner.Run(ctx, region, job) - if err != nil { - jobError := NewError(jobContext, ResourceMetadataErr, err) - mux.Lock() - jobErrors = append(jobErrors, jobError) - mux.Unlock() - - return - } - if len(resources) > 0 { - result := model.TaggedResourceResult{ - Context: jobContext.ToScrapeContext(job.CustomTags), - Data: resources, - } - mux.Lock() - resourceResults = append(resourceResults, result) - mux.Unlock() - } else { - jobLogger.Debug("No tagged resources") - } - jobLogger.Debug("Resource discovery finished", "number_of_discovered_resources", len(resources)) - - jobToRun = cloudwatchrunner.DiscoveryJob{Job: job, Resources: resources} - }, func(job model.CustomNamespaceJob) { - jobToRun = cloudwatchrunner.CustomNamespaceJob{Job: job} - }, - ) - if jobToRun == nil { - jobLogger.Debug("Ending job run early due to job error see job errors") - return - } - - jobLogger.Debug("Starting cloudwatch metrics runner") - cwRunner := s.runnerFactory.NewCloudWatchRunner(jobLogger, region, role, jobToRun) - metricResult, err := cwRunner.Run(ctx) - if err != nil { - jobError := NewError(jobContext, CloudWatchCollectionErr, err) - mux.Lock() - jobErrors = append(jobErrors, jobError) - mux.Unlock() - - return - } - - if len(metricResult) == 0 { - jobLogger.Debug("No metrics data found") - return - } - - jobLogger.Debug("Job run finished", "number_of_metrics", len(metricResult)) - - result := model.CloudwatchMetricResult{ - Context: jobContext.ToScrapeContext(jobToRun.CustomTags()), - Data: metricResult, - } - - mux.Lock() - defer mux.Unlock() - metricResults = append(metricResults, result) - }() - }) - wg.Wait() - s.logger.Debug("Finished job runs", "resource_results", len(resourceResults), "metric_results", len(metricResults)) - return resourceResults, metricResults, jobErrors -} - -// Walk through each custom namespace and discovery jobs and take an action -func jobConfigVisitor(jobsCfg model.JobsConfig, action func(job any, role model.Role, region string)) { - for _, job := range jobsCfg.DiscoveryJobs { - for _, role := range job.Roles { - for _, region := range job.Regions { - action(job, role, region) - } - } - } - - for _, job := range jobsCfg.CustomNamespaceJobs { - for _, role := range job.Roles { - for _, region := range job.Regions { - action(job, role, region) - } - } - } -} - -// Take an action depending on the job type, only supports discovery and custom job types -func jobAction(logger *slog.Logger, job any, discovery func(job model.DiscoveryJob), custom func(job model.CustomNamespaceJob)) { - // Type switches are free https://stackoverflow.com/a/28027945 - switch typedJob := job.(type) { - case model.DiscoveryJob: - discovery(typedJob) - case model.CustomNamespaceJob: - custom(typedJob) - default: - logger.Error("Unexpected job type", "err", fmt.Errorf("config type of %T is not supported", typedJob)) - return - } -} - -// JobContext exists to track data we want for logging, errors, or other output context that's learned as the job runs -// This makes it easier to track the data additively and morph it to the final shape necessary be it a model.ScrapeContext -// or an Error. It's an exported type for tests but is not part of the public interface -type JobContext struct { //nolint:revive - Account Account - Namespace string - Region string - RoleARN string -} - -func (jc JobContext) ToScrapeContext(customTags []model.Tag) *model.ScrapeContext { - return &model.ScrapeContext{ - AccountID: jc.Account.ID, - Region: jc.Region, - CustomTags: customTags, - AccountAlias: jc.Account.Alias, - } -} - -type Error struct { - JobContext - ErrorType ErrorType - Err error -} - -func NewError(context JobContext, errorType ErrorType, err error) Error { - return Error{ - JobContext: context, - ErrorType: errorType, - Err: err, - } -} - -func (e Error) ToLoggerKeyVals() []interface{} { - return []interface{}{ - "account_id", e.Account.ID, - "namespace", e.Namespace, - "region", e.Region, - "role_arn", e.RoleARN, - } -} diff --git a/pkg/job/scraper_test.go b/pkg/job/scraper_test.go deleted file mode 100644 index 960280850..000000000 --- a/pkg/job/scraper_test.go +++ /dev/null @@ -1,565 +0,0 @@ -// Copyright 2024 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package job_test - -import ( - "context" - "errors" - "log/slog" - "reflect" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/prometheus/common/promslog" - "github.com/r3labs/diff/v3" - "github.com/stretchr/testify/assert" - - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/clients/account" - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/job" - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/job/cloudwatchrunner" - "github.com/prometheus-community/yet-another-cloudwatch-exporter/pkg/model" -) - -type testRunnerFactory struct { - GetAccountAliasFunc func() (string, error) - GetAccountFunc func() (string, error) - MetadataRunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) - CloudwatchRunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) -} - -func (t *testRunnerFactory) GetAccountAlias(context.Context) (string, error) { - return t.GetAccountAliasFunc() -} - -func (t *testRunnerFactory) GetAccount(context.Context) (string, error) { - return t.GetAccountFunc() -} - -func (t *testRunnerFactory) Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) { - return t.MetadataRunFunc(ctx, region, job) -} - -func (t *testRunnerFactory) GetAccountClient(string, model.Role) account.Client { - return t -} - -func (t *testRunnerFactory) NewResourceMetadataRunner(*slog.Logger, string, model.Role) job.ResourceMetadataRunner { - return &testMetadataRunner{RunFunc: t.MetadataRunFunc} -} - -func (t *testRunnerFactory) NewCloudWatchRunner(_ *slog.Logger, _ string, _ model.Role, job cloudwatchrunner.Job) job.CloudwatchRunner { - return &testCloudwatchRunner{Job: job, RunFunc: t.CloudwatchRunFunc} -} - -type testMetadataRunner struct { - RunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) -} - -func (t testMetadataRunner) Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) { - return t.RunFunc(ctx, region, job) -} - -type testCloudwatchRunner struct { - RunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) - Job cloudwatchrunner.Job -} - -func (t testCloudwatchRunner) Run(ctx context.Context) ([]*model.CloudwatchData, error) { - return t.RunFunc(ctx, t.Job) -} - -func TestScrapeRunner_Run(t *testing.T) { - tests := []struct { - name string - jobsCfg model.JobsConfig - getAccountFunc func() (string, error) - getAccountAliasFunc func() (string, error) - metadataRunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) - cloudwatchRunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) - expectedResources []model.TaggedResourceResult - expectedMetrics []model.CloudwatchMetricResult - expectedErrs []job.Error - }{ - { - name: "can run a discovery job", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { - return "my-aws-account", nil - }, - metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { - return []*model.TaggedResource{{ - ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - }}, nil - }, - cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - return []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedResources: []model.TaggedResourceResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.TaggedResource{ - {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, - }, - }, - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - }, - { - name: "can run a custom namespace job", - jobsCfg: model.JobsConfig{ - CustomNamespaceJobs: []model.CustomNamespaceJob{ - { - Regions: []string{"us-east-2"}, - Name: "my-custom-job", - Namespace: "custom-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { - return "my-aws-account", nil - }, - cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - return []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - }, - { - name: "can run a discovery and custom namespace job", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - CustomNamespaceJobs: []model.CustomNamespaceJob{ - { - Regions: []string{"us-east-2"}, - Name: "my-custom-job", - Namespace: "custom-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { - return "my-aws-account", nil - }, - metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { - return []*model.TaggedResource{{ - ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - }}, nil - }, - cloudwatchRunFunc: func(_ context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - if job.Namespace() == "custom-namespace" { - return []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, nil - } - return []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedResources: []model.TaggedResourceResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.TaggedResource{ - {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, - }, - }, - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, - }, - { - Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - }, - { - name: "returns errors from GetAccounts", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - CustomNamespaceJobs: []model.CustomNamespaceJob{ - { - Regions: []string{"us-east-2"}, - Name: "my-custom-job", - Namespace: "custom-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "", errors.New("failed to get account") - }, - expectedErrs: []job.Error{ - {JobContext: job.JobContext{Account: job.Account{}, Namespace: "aws-namespace", Region: "us-east-1", RoleARN: "aws-arn-1"}, ErrorType: job.AccountErr}, - {JobContext: job.JobContext{Account: job.Account{}, Namespace: "custom-namespace", Region: "us-east-2", RoleARN: "aws-arn-2"}, ErrorType: job.AccountErr}, - }, - }, - { - name: "ignores errors from GetAccountAlias", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { return "", errors.New("No alias here") }, - metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { - return []*model.TaggedResource{{ - ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - }}, nil - }, - cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - return []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedResources: []model.TaggedResourceResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: ""}, - Data: []*model.TaggedResource{ - {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, - }, - }, - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: ""}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - }, - { - name: "returns errors from resource discovery without failing scrape", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - CustomNamespaceJobs: []model.CustomNamespaceJob{ - { - Regions: []string{"us-east-2"}, - Name: "my-custom-job", - Namespace: "custom-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { - return "my-aws-account", nil - }, - metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { - return nil, errors.New("I failed you") - }, - cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - return []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-2", - ResourceName: "resource-2", - Namespace: "custom-namespace", - Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", DataPoints: []model.DataPoint{{Value: aws.Float64(2.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - expectedErrs: []job.Error{ - { - JobContext: job.JobContext{ - Account: job.Account{ID: "aws-account-1", Alias: "my-aws-account"}, - Namespace: "aws-namespace", - Region: "us-east-1", - RoleARN: "aws-arn-1", - }, - ErrorType: job.ResourceMetadataErr, - }, - }, - }, - { - name: "returns errors from cloudwatch metrics runner without failing scrape", - jobsCfg: model.JobsConfig{ - DiscoveryJobs: []model.DiscoveryJob{ - { - Regions: []string{"us-east-1"}, - Namespace: "aws-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, - }, - }, - }, - CustomNamespaceJobs: []model.CustomNamespaceJob{ - { - Regions: []string{"us-east-2"}, - Name: "my-custom-job", - Namespace: "custom-namespace", - Roles: []model.Role{ - {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, - }, - }, - }, - }, - getAccountFunc: func() (string, error) { - return "aws-account-1", nil - }, - getAccountAliasFunc: func() (string, error) { - return "my-aws-account", nil - }, - metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { - return []*model.TaggedResource{{ - ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - }}, nil - }, - cloudwatchRunFunc: func(_ context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { - if job.Namespace() == "custom-namespace" { - return nil, errors.New("I failed you") - } - return []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, nil - }, - expectedResources: []model.TaggedResourceResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.TaggedResource{ - {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, - }, - }, - }, - expectedMetrics: []model.CloudwatchMetricResult{ - { - Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, - Data: []*model.CloudwatchData{ - { - MetricName: "metric-1", - ResourceName: "resource-1", - Namespace: "aws-namespace", - Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, - Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, - GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", DataPoints: []model.DataPoint{{Value: aws.Float64(1.0), Timestamp: time.Time{}}}}, - }, - }, - }, - }, - expectedErrs: []job.Error{ - { - JobContext: job.JobContext{ - Account: job.Account{ID: "aws-account-1", Alias: "my-aws-account"}, - Namespace: "custom-namespace", - Region: "us-east-2", - RoleARN: "aws-arn-2", - }, - ErrorType: job.CloudWatchCollectionErr, - }, - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - rf := testRunnerFactory{ - GetAccountFunc: tc.getAccountFunc, - GetAccountAliasFunc: tc.getAccountAliasFunc, - MetadataRunFunc: tc.metadataRunFunc, - CloudwatchRunFunc: tc.cloudwatchRunFunc, - } - lvl := promslog.NewLevel() - _ = lvl.Set("debug") - sr := job.NewScraper(promslog.New(&promslog.Config{Level: lvl}), tc.jobsCfg, &rf) - resources, metrics, errs := sr.Scrape(context.Background()) - - changelog, err := diff.Diff(tc.expectedResources, resources) - assert.NoError(t, err, "failed to diff resources") - assert.Len(t, changelog, 0, changelog) - - changelog, err = diff.Diff(tc.expectedMetrics, metrics) - assert.NoError(t, err, "failed to diff metrics") - assert.Len(t, changelog, 0, changelog) - - // We don't want to check the exact error just the message - changelog, err = diff.Diff(tc.expectedErrs, errs, diff.Filter(func(_ []string, _ reflect.Type, field reflect.StructField) bool { - return field.Name != "Err" - })) - assert.NoError(t, err, "failed to diff errs") - assert.Len(t, changelog, 0, changelog) - }) - } -}