From 7c625b7e4efbcf34f0d7b0074b76eaaa0ae22de9 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Wed, 15 Feb 2023 15:26:33 +0530 Subject: [PATCH 1/3] add unit test for BucketList function --- metadata/pkg/drivers/aws/aws_test.go | 625 +++++++++++++++++++++++++++ 1 file changed, 625 insertions(+) create mode 100644 metadata/pkg/drivers/aws/aws_test.go diff --git a/metadata/pkg/drivers/aws/aws_test.go b/metadata/pkg/drivers/aws/aws_test.go new file mode 100644 index 000000000..593e3dcd2 --- /dev/null +++ b/metadata/pkg/drivers/aws/aws_test.go @@ -0,0 +1,625 @@ +package aws + +import ( + "errors" + "testing" + "time" + + "github.com/opensds/multi-cloud/metadata/pkg/model" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/stretchr/testify/assert" +) + +type MockS3Client struct { + s3iface.S3API + ListBucketsResp *s3.ListBucketsOutput + ListBucketsError error + GetBucketLocationResp *s3.GetBucketLocationOutput + GetBucketLocationError error + GetBucketTaggingResp *s3.GetBucketTaggingOutput + GetBucketTaggingError error + GetBucketAclResp *s3.GetBucketAclOutput + GetBucketAclError error + ListObjectsV2Resp *s3.ListObjectsV2Output + ListObjectsV2Error error + GetObjectTaggingResp *s3.GetObjectTaggingOutput + GetObjectTaggingError error + GetObjectAclResp *s3.GetObjectAclOutput + GetObjectAclError error + HeadObjectResp *s3.HeadObjectOutput + HeadObjectError error +} + +type Test struct { + TestNumber int + ListBucketsResp *s3.ListBucketsOutput + ListBucketsError error + GetBucketLocationResp *s3.GetBucketLocationOutput + GetBucketLocationError error + GetBucketTaggingResp *s3.GetBucketTaggingOutput + GetBucketTaggingError error + GetBucketAclResp *s3.GetBucketAclOutput + GetBucketAclError error + ListObjectsV2Resp *s3.ListObjectsV2Output + ListObjectsV2Error error + GetObjectTaggingResp *s3.GetObjectTaggingOutput + GetObjectTaggingError error + GetObjectAclResp *s3.GetObjectAclOutput + GetObjectAclError error + HeadObjectResp *s3.HeadObjectOutput + HeadObjectError error + ExpectedResp []*model.MetaBucket + ExpectedError error +} + +var TIMEVAL = time.Date(2023, time.February, 12, 9, 23, 34, 45, time.UTC) + +func TestBucketList(t *testing.T) { + tests := []Test{ + { + TestNumber: 1, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingResp: &s3.GetBucketTaggingOutput{TagSet: []*s3.Tag{ + {Key: aws.String("test1"), Value: aws.String("test2")}, + {Key: aws.String("sample1"), Value: aws.String("sample2")}, + }}, + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object2"), LastModified: &TIMEVAL, Size: aws.Int64(425), StorageClass: aws.String("GLACIER_IR")}, + {Key: aws.String("object3"), LastModified: &TIMEVAL, Size: aws.Int64(435), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetBucketAclError: errors.New("GetBucketAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 3, + TotalSize: 1275, + BucketTags: map[string]string{"test1": "test2", "sample1": "sample2"}, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + { + ObjectName: "object2", + LastModifiedDate: &TIMEVAL, + Size: 425, + StorageClass: "GLACIER_IR", + }, + { + ObjectName: "object3", + LastModifiedDate: &TIMEVAL, + Size: 435, + StorageClass: "STANDARD", + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 2, + ListBucketsError: errors.New("ListBucketsForbidden"), + + ExpectedResp: nil, + ExpectedError: errors.New("ListBucketsForbidden"), + }, + + { + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + {CreationDate: &TIMEVAL, Name: aws.String("ghi")}, + }, + }, + GetBucketLocationError: errors.New("GetBucketLocationError"), + + ExpectedResp: []*model.MetaBucket{}, + ExpectedError: nil, + }, + + { + TestNumber: 3, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(0), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object2"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 2, + TotalSize: 415, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 0, + StorageClass: "STANDARD", + }, + { + ObjectName: "object2", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 4, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingResp: &s3.GetBucketTaggingOutput{}, + GetBucketTaggingError: errors.New("NoSuchTagSet"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(1533), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 1, + TotalSize: 1533, + BucketTags: map[string]string{}, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 1533, + StorageClass: "STANDARD", + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 5, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 6, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + {CreationDate: &TIMEVAL, Name: aws.String("ghi")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-northeast-1")}, + + ExpectedResp: []*model.MetaBucket{}, + ExpectedError: nil, + }, + + { + TestNumber: 7, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("ghi")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectResp: &s3.HeadObjectOutput{ + ServerSideEncryption: aws.String("AE256"), + ContentType: aws.String("msword"), + Expires: aws.String("2023-02-12T09:23:34.000000045Z"), + ReplicationStatus: aws.String("confirmed"), + WebsiteRedirectLocation: aws.String("/something"), + Metadata: map[string]*string{ + "k1": aws.String("v1"), + "k2": aws.String("v2"), + }}, + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "ghi", + Region: "ap-south-1", + NumberOfObjects: 2, + TotalSize: 830, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ServerSideEncryption: "AE256", + ObjectType: "msword", + ExpiresDate: &TIMEVAL, + ReplicationStatus: "confirmed", + RedirectLocation: "/something", + Metadata: map[string]string{"k1": "v1", "k2": "v2"}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ServerSideEncryption: "AE256", + ObjectType: "msword", + ExpiresDate: &TIMEVAL, + ReplicationStatus: "confirmed", + RedirectLocation: "/something", + Metadata: map[string]string{"k1": "v1", "k2": "v2"}, + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 8, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("vui")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectResp: &s3.HeadObjectOutput{ + Expires: aws.String("2022-11-.000Z"), + WebsiteRedirectLocation: aws.String("/something"), + }, + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "vui", + Region: "ap-south-1", + NumberOfObjects: 2, + TotalSize: 830, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 9, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Error: errors.New("ListObjectsForbidden"), + + ExpectedResp: []*model.MetaBucket{}, + ExpectedError: nil, + }, + + { + TestNumber: 10, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("xyz")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectTaggingResp: &s3.GetObjectTaggingOutput{TagSet: []*s3.Tag{{Key: aws.String("key1"), Value: aws.String("val1")}}, VersionId: aws.String("dubwvnkewv.fewvbib")}, + GetObjectAclError: errors.New("GetObjectAclError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "xyz", + Region: "ap-south-1", + NumberOfObjects: 2, + TotalSize: 830, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectTags: map[string]string{"key1": "val1"}, + VersionId: "dubwvnkewv.fewvbib", + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectTags: map[string]string{"key1": "val1"}, + VersionId: "dubwvnkewv.fewvbib", + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 11, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + GetBucketAclResp: &s3.GetBucketAclOutput{Grants: []*s3.Grant{{Grantee: &s3.Grantee{ID: aws.String("ewwiubibu")}, Permission: aws.String("FULL_CONTROL")}}}, + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetObjectAclResp: &s3.GetObjectAclOutput{Grants: []*s3.Grant{{Grantee: &s3.Grantee{ID: aws.String("ewwiubibu")}, Permission: aws.String("FULL_CONTROL")}}}, + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 1, + TotalSize: 415, + BucketAcl: []*model.Access{{ID: "ewwiubibu", Permission: "FULL_CONTROL"}}, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectAcl: []*model.Access{{ID: "ewwiubibu", Permission: "FULL_CONTROL"}}, + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 12, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + {CreationDate: &TIMEVAL, Name: aws.String("qqq")}, + {CreationDate: &TIMEVAL, Name: aws.String("ghi")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 1, + TotalSize: 415, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + }, + }, + { + CreationDate: &TIMEVAL, + Name: "ghi", + Region: "ap-south-1", + NumberOfObjects: 1, + TotalSize: 415, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + }, + }, + }, + ExpectedError: nil, + }, + + { + TestNumber: 13, + ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ + {CreationDate: &TIMEVAL, Name: aws.String("abc")}, + {CreationDate: &TIMEVAL, Name: aws.String("qqq")}, + {CreationDate: &TIMEVAL, Name: aws.String("ppp")}, + }, + }, + GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, + GetBucketTaggingError: errors.New("GetBucketTaggingError"), + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), + + ExpectedResp: []*model.MetaBucket{ + { + CreationDate: &TIMEVAL, + Name: "abc", + Region: "ap-south-1", + NumberOfObjects: 1, + TotalSize: 415, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + }, + }, + }, + ExpectedError: nil, + }, + } + + for _, test := range tests { + backendRegion := "ap-south-1" + mockSvc := &MockS3Client{ + ListBucketsResp: test.ListBucketsResp, + ListBucketsError: test.ListBucketsError, + GetBucketLocationResp: test.GetBucketLocationResp, + GetBucketLocationError: test.GetBucketLocationError, + HeadObjectResp: test.HeadObjectResp, + HeadObjectError: test.HeadObjectError, + GetObjectAclResp: test.GetObjectAclResp, + GetObjectAclError: test.GetObjectAclError, + GetBucketTaggingResp: test.GetBucketTaggingResp, + GetBucketTaggingError: test.GetBucketTaggingError, + ListObjectsV2Resp: test.ListObjectsV2Resp, + ListObjectsV2Error: test.ListObjectsV2Error, + GetBucketAclResp: test.GetBucketAclResp, + GetBucketAclError: test.GetBucketAclError, + GetObjectTaggingResp: test.GetObjectTaggingResp, + GetObjectTaggingError: test.GetObjectTaggingError, + } + bucketArray, err := BucketList(mockSvc, &backendRegion) + + assert.Equal(t, test.ExpectedResp, bucketArray, test.TestNumber) + assert.Equal(t, test.ExpectedError, err, test.TestNumber) + } +} + +func (m *MockS3Client) ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) { + return m.ListBucketsResp, m.ListBucketsError +} + +func (m *MockS3Client) GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) { + if *input.Bucket == "qqq" { + return &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-northeast-1")}, m.GetBucketLocationError + } + return m.GetBucketLocationResp, m.GetBucketLocationError +} + +func (m *MockS3Client) GetBucketTagging(input *s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error) { + return m.GetBucketTaggingResp, m.GetBucketTaggingError +} + +func (m *MockS3Client) GetBucketAcl(input *s3.GetBucketAclInput) (*s3.GetBucketAclOutput, error) { + return m.GetBucketAclResp, m.GetBucketAclError +} + +func (m *MockS3Client) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + if *input.Bucket == "ppp" { + return nil, errors.New("ListObjectsV2Error") + } + return m.ListObjectsV2Resp, m.ListObjectsV2Error +} + +func (m *MockS3Client) GetObjectTagging(*s3.GetObjectTaggingInput) (*s3.GetObjectTaggingOutput, error) { + return m.GetObjectTaggingResp, m.GetObjectTaggingError +} + +func (m *MockS3Client) GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { + return m.GetObjectAclResp, m.GetObjectAclError +} + +func (m *MockS3Client) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + return m.HeadObjectResp, m.HeadObjectError +} From 22af6aed16e1de0d9bc7ae798090e083b6818c73 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Tue, 21 Feb 2023 00:01:21 +0530 Subject: [PATCH 2/3] add unit test for ObjectList function --- metadata/pkg/drivers/aws/aws.go | 54 +-- metadata/pkg/drivers/aws/aws_test.go | 617 ++++++++++++++------------- 2 files changed, 358 insertions(+), 313 deletions(-) diff --git a/metadata/pkg/drivers/aws/aws.go b/metadata/pkg/drivers/aws/aws.go index 4fdaa63e6..90138e158 100755 --- a/metadata/pkg/drivers/aws/aws.go +++ b/metadata/pkg/drivers/aws/aws.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" backendpb "github.com/opensds/multi-cloud/backend/proto" "github.com/opensds/multi-cloud/metadata/pkg/db" "github.com/opensds/multi-cloud/metadata/pkg/model" @@ -50,8 +51,7 @@ type AwsAdapter struct { Session *session.Session } -func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObject) { - svc := s3.New(sess) +func GetHeadObject(svc s3iface.S3API, bucketName *string, obj *model.MetaObject) { meta, err := svc.HeadObject(&s3.HeadObjectInput{Bucket: bucketName, Key: &obj.ObjectName}) if err != nil { log.Errorf("cannot perform head object on object %v in bucket %v. failed with error: %v", obj.ObjectName, *bucketName, err) @@ -60,7 +60,9 @@ func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObj if meta.ServerSideEncryption != nil { obj.ServerSideEncryption = *meta.ServerSideEncryption } - obj.ObjectType = *meta.ContentType + if meta.ContentType != nil { + obj.ObjectType = *meta.ContentType + } if meta.Expires != nil { expiresTime, err := time.Parse(time.RFC3339, *meta.Expires) if err != nil { @@ -82,12 +84,11 @@ func GetHeadObject(sess *session.Session, bucketName *string, obj *model.MetaObj obj.Metadata = metadata } -func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { - svc := s3.New(sess) - output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket.Name}) +var ObjectList = func(svc s3iface.S3API, bucketName string) ([]*model.MetaObject, int64, error) { + output, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucketName}) if err != nil { - log.Errorf("unable to list objects in bucket %v. failed with error: %v", bucket.Name, err) - return err + log.Errorf("unable to list objects in bucket %v. failed with error: %v", bucketName, err) + return nil, 0, err } numObjects := len(output.Contents) @@ -102,7 +103,7 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { totSize += obj.Size obj.StorageClass = *object.StorageClass - tags, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{Bucket: &bucket.Name, Key: &obj.ObjectName}) + tags, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{Bucket: &bucketName, Key: &obj.ObjectName}) if err == nil { tagset := map[string]string{} @@ -117,7 +118,7 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { log.Errorf("unable to get object tags. failed with error: %v", err) } - acl, err := svc.GetObjectAcl(&s3.GetObjectAclInput{Bucket: &bucket.Name, Key: &obj.ObjectName}) + acl, err := svc.GetObjectAcl(&s3.GetObjectAclInput{Bucket: &bucketName, Key: &obj.ObjectName}) if err != nil { log.Errorf("unable to get object Acl. failed with error: %v", err) } else { @@ -128,25 +129,21 @@ func ObjectList(sess *session.Session, bucket *model.MetaBucket) error { obj.ObjectAcl = access } - GetHeadObject(sess, &bucket.Name, obj) + GetHeadObject(svc, &bucketName, obj) } - bucket.NumberOfObjects = numObjects - bucket.TotalSize = totSize - bucket.Objects = objectArray - return nil + return objectArray, totSize, nil } -func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { +func GetBucketMeta(buckIdx int, bucket *s3.Bucket, svc s3iface.S3API, backendRegion *string, bucketArray []*model.MetaBucket, wg *sync.WaitGroup) { defer wg.Done() - svc := s3.New(sess) loc, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: bucket.Name}) if err != nil { log.Errorf("unable to get bucket location. failed with error: %v", err) return } - if *loc.LocationConstraint != *sess.Config.Region { + if *loc.LocationConstraint != *backendRegion { return } @@ -155,22 +152,27 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket buck.CreationDate = bucket.CreationDate buck.Region = *loc.LocationConstraint - err = ObjectList(sess, buck) + objectArray, totalSize, err := ObjectList(svc, *bucket.Name) + if err != nil { return } + buck.Objects = objectArray + buck.NumberOfObjects = len(objectArray) + buck.TotalSize = totalSize + bucketArray[buckIdx] = buck tags, err := svc.GetBucketTagging(&s3.GetBucketTaggingInput{Bucket: bucket.Name}) - if err == nil { + if err == nil || strings.Contains(err.Error(), "NoSuchTagSet") { tagset := map[string]string{} for _, tag := range tags.TagSet { tagset[*tag.Key] = *tag.Value } buck.BucketTags = tagset - } else if !strings.Contains(err.Error(), "NoSuchTagSet") { + } else { log.Errorf("unable to get bucket tags. failed with error: %v", err) } @@ -186,8 +188,7 @@ func GetBucketMeta(buckIdx int, bucket *s3.Bucket, sess *session.Session, bucket } } -func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { - svc := s3.New(sess) +func BucketList(svc s3iface.S3API, backendRegion *string) ([]*model.MetaBucket, error) { output, err := svc.ListBuckets(&s3.ListBucketsInput{}) if err != nil { @@ -198,7 +199,7 @@ func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { wg := sync.WaitGroup{} for idx, bucket := range output.Buckets { wg.Add(1) - go GetBucketMeta(idx, bucket, sess, bucketArray, &wg) + go GetBucketMeta(idx, bucket, svc, backendRegion, bucketArray, &wg) } wg.Wait() @@ -213,8 +214,9 @@ func BucketList(sess *session.Session) ([]*model.MetaBucket, error) { } func (ad *AwsAdapter) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest) error { - - buckArr, err := BucketList(ad.Session) + svc := s3.New(ad.Session) + backendRegion := ad.Session.Config.Region + buckArr, err := BucketList(svc, backendRegion) if err != nil { log.Errorf("metadata collection for backend id: %v failed with error: %v", ad.Backend.Id, err) return err diff --git a/metadata/pkg/drivers/aws/aws_test.go b/metadata/pkg/drivers/aws/aws_test.go index 593e3dcd2..a0e80ddb0 100644 --- a/metadata/pkg/drivers/aws/aws_test.go +++ b/metadata/pkg/drivers/aws/aws_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 The SODA Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package aws import ( @@ -13,6 +27,13 @@ import ( "github.com/stretchr/testify/assert" ) +func MockObjectList(svc s3iface.S3API, bucketName string) ([]*model.MetaObject, int64, error) { + if bucketName == "ppp" { + return nil, 0, errors.New("ListObjectsV2Error") + } + return []*model.MetaObject{}, 0, nil +} + type MockS3Client struct { s3iface.S3API ListBucketsResp *s3.ListBucketsOutput @@ -31,6 +52,7 @@ type MockS3Client struct { GetObjectAclError error HeadObjectResp *s3.HeadObjectOutput HeadObjectError error + BucketName string } type Test struct { @@ -51,12 +73,272 @@ type Test struct { GetObjectAclError error HeadObjectResp *s3.HeadObjectOutput HeadObjectError error - ExpectedResp []*model.MetaBucket + ExpectedBucketResp []*model.MetaBucket + ExpectedObjectResp []*model.MetaObject ExpectedError error + BucketName string + ExpectedBucketSize int64 } var TIMEVAL = time.Date(2023, time.February, 12, 9, 23, 34, 45, time.UTC) +func TestObjectList(t *testing.T) { + tests := []Test{ + { + TestNumber: 1, + BucketName: "abc", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object2"), LastModified: &TIMEVAL, Size: aws.Int64(425), StorageClass: aws.String("GLACIER_IR")}, + {Key: aws.String("object3"), LastModified: &TIMEVAL, Size: aws.Int64(435), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + }, + { + ObjectName: "object2", + LastModifiedDate: &TIMEVAL, + Size: 425, + StorageClass: "GLACIER_IR", + }, + { + ObjectName: "object3", + LastModifiedDate: &TIMEVAL, + Size: 435, + StorageClass: "STANDARD", + }, + }, + ExpectedBucketSize: 1275, + ExpectedError: nil, + }, + + { + TestNumber: 2, + BucketName: "abc", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(1533), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 1533, + StorageClass: "STANDARD", + }, + }, + ExpectedBucketSize: 1533, + ExpectedError: nil, + }, + + { + TestNumber: 3, + BucketName: "abc", + ListObjectsV2Resp: &s3.ListObjectsV2Output{}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: []*model.MetaObject{}, + ExpectedBucketSize: 0, + ExpectedError: nil, + }, + + { + TestNumber: 4, + BucketName: "ghi", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectResp: &s3.HeadObjectOutput{ + ServerSideEncryption: aws.String("AE256"), + ContentType: aws.String("msword"), + Expires: aws.String("2023-02-12T09:23:34.000000045Z"), + ReplicationStatus: aws.String("confirmed"), + WebsiteRedirectLocation: aws.String("/something"), + Metadata: map[string]*string{ + "k1": aws.String("v1"), + "k2": aws.String("v2"), + }}, + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ServerSideEncryption: "AE256", + ObjectType: "msword", + ExpiresDate: &TIMEVAL, + ReplicationStatus: "confirmed", + RedirectLocation: "/something", + Metadata: map[string]string{"k1": "v1", "k2": "v2"}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ServerSideEncryption: "AE256", + ObjectType: "msword", + ExpiresDate: &TIMEVAL, + ReplicationStatus: "confirmed", + RedirectLocation: "/something", + Metadata: map[string]string{"k1": "v1", "k2": "v2"}, + }, + }, + ExpectedBucketSize: 830, + ExpectedError: nil, + }, + + { + TestNumber: 5, + BucketName: "vui", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectResp: &s3.HeadObjectOutput{ + Expires: aws.String("2022-11-.000Z"), + WebsiteRedirectLocation: aws.String("/something"), + }, + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + }, + ExpectedBucketSize: 830, + ExpectedError: nil, + }, + + { + TestNumber: 6, + BucketName: "abc", + ListObjectsV2Error: errors.New("ListObjectsForbidden"), + + ExpectedObjectResp: nil, + ExpectedError: errors.New("ListObjectsForbidden"), + }, + + { + TestNumber: 7, + BucketName: "xyz", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectTaggingResp: &s3.GetObjectTaggingOutput{TagSet: []*s3.Tag{{Key: aws.String("key1"), Value: aws.String("val1")}}, VersionId: aws.String("dubwvnkewv.fewvbib")}, + GetObjectAclError: errors.New("GetObjectAclError"), + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectTags: map[string]string{"key1": "val1"}, + VersionId: "dubwvnkewv.fewvbib", + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectTags: map[string]string{"key1": "val1"}, + VersionId: "dubwvnkewv.fewvbib", + }, + }, + ExpectedBucketSize: 830, + ExpectedError: nil, + }, + + { + TestNumber: 8, + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetObjectAclResp: &s3.GetObjectAclOutput{Grants: []*s3.Grant{{Grantee: &s3.Grantee{ID: aws.String("ewwiubibu")}, Permission: aws.String("FULL_CONTROL")}}}, + + ExpectedObjectResp: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + ObjectAcl: []*model.Access{{ID: "ewwiubibu", Permission: "FULL_CONTROL"}}, + }, + }, + ExpectedBucketSize: 415, + ExpectedError: nil, + }, + + { + TestNumber: 9, + BucketName: "ppp", + ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ + {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, + }}, + HeadObjectError: errors.New("HeadObjectError"), + GetObjectAclError: errors.New("GetObjectAclError"), + GetObjectTaggingError: errors.New("GetObjectTaggingError"), + + ExpectedObjectResp: nil, + ExpectedBucketSize: 0, + ExpectedError: errors.New("ListObjectsV2Error"), + }, + } + for _, test := range tests { + mockSvc := &MockS3Client{ + HeadObjectResp: test.HeadObjectResp, + HeadObjectError: test.HeadObjectError, + GetObjectAclResp: test.GetObjectAclResp, + GetObjectAclError: test.GetObjectAclError, + ListObjectsV2Resp: test.ListObjectsV2Resp, + ListObjectsV2Error: test.ListObjectsV2Error, + GetObjectTaggingResp: test.GetObjectTaggingResp, + GetObjectTaggingError: test.GetObjectTaggingError, + BucketName: test.BucketName, + } + objectArray, totalSize, err := ObjectList(mockSvc, mockSvc.BucketName) + assert.Equal(t, test.ExpectedObjectResp, objectArray, test.TestNumber) + assert.Equal(t, test.ExpectedBucketSize, totalSize, test.TestNumber) + assert.Equal(t, test.ExpectedError, err, test.TestNumber) + } +} + func TestBucketList(t *testing.T) { tests := []Test{ { @@ -70,44 +352,17 @@ func TestBucketList(t *testing.T) { {Key: aws.String("test1"), Value: aws.String("test2")}, {Key: aws.String("sample1"), Value: aws.String("sample2")}, }}, - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - {Key: aws.String("object2"), LastModified: &TIMEVAL, Size: aws.Int64(425), StorageClass: aws.String("GLACIER_IR")}, - {Key: aws.String("object3"), LastModified: &TIMEVAL, Size: aws.Int64(435), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetBucketAclError: errors.New("GetBucketAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), + GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 3, - TotalSize: 1275, + NumberOfObjects: 0, + TotalSize: 0, BucketTags: map[string]string{"test1": "test2", "sample1": "sample2"}, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - }, - { - ObjectName: "object2", - LastModifiedDate: &TIMEVAL, - Size: 425, - StorageClass: "GLACIER_IR", - }, - { - ObjectName: "object3", - LastModifiedDate: &TIMEVAL, - Size: 435, - StorageClass: "STANDARD", - }, - }, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -117,8 +372,8 @@ func TestBucketList(t *testing.T) { TestNumber: 2, ListBucketsError: errors.New("ListBucketsForbidden"), - ExpectedResp: nil, - ExpectedError: errors.New("ListBucketsForbidden"), + ExpectedBucketResp: nil, + ExpectedError: errors.New("ListBucketsForbidden"), }, { @@ -129,8 +384,8 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationError: errors.New("GetBucketLocationError"), - ExpectedResp: []*model.MetaBucket{}, - ExpectedError: nil, + ExpectedBucketResp: []*model.MetaBucket{}, + ExpectedError: nil, }, { @@ -141,36 +396,16 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(0), StorageClass: aws.String("STANDARD")}, - {Key: aws.String("object2"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 2, - TotalSize: 415, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 0, - StorageClass: "STANDARD", - }, - { - ObjectName: "object2", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -185,30 +420,17 @@ func TestBucketList(t *testing.T) { GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingResp: &s3.GetBucketTaggingOutput{}, GetBucketTaggingError: errors.New("NoSuchTagSet"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(1533), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 1, - TotalSize: 1533, + NumberOfObjects: 0, + TotalSize: 0, BucketTags: map[string]string{}, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 1533, - StorageClass: "STANDARD", - }, - }, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -222,13 +444,9 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", @@ -250,8 +468,8 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-northeast-1")}, - ExpectedResp: []*model.MetaBucket{}, - ExpectedError: nil, + ExpectedBucketResp: []*model.MetaBucket{}, + ExpectedError: nil, }, { @@ -262,57 +480,16 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectResp: &s3.HeadObjectOutput{ - ServerSideEncryption: aws.String("AE256"), - ContentType: aws.String("msword"), - Expires: aws.String("2023-02-12T09:23:34.000000045Z"), - ReplicationStatus: aws.String("confirmed"), - WebsiteRedirectLocation: aws.String("/something"), - Metadata: map[string]*string{ - "k1": aws.String("v1"), - "k2": aws.String("v2"), - }}, - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "ghi", Region: "ap-south-1", - NumberOfObjects: 2, - TotalSize: 830, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - ServerSideEncryption: "AE256", - ObjectType: "msword", - ExpiresDate: &TIMEVAL, - ReplicationStatus: "confirmed", - RedirectLocation: "/something", - Metadata: map[string]string{"k1": "v1", "k2": "v2"}, - }, - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - ServerSideEncryption: "AE256", - ObjectType: "msword", - ExpiresDate: &TIMEVAL, - ReplicationStatus: "confirmed", - RedirectLocation: "/something", - Metadata: map[string]string{"k1": "v1", "k2": "v2"}, - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -326,43 +503,16 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectResp: &s3.HeadObjectOutput{ - Expires: aws.String("2022-11-.000Z"), - WebsiteRedirectLocation: aws.String("/something"), - }, - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "vui", Region: "ap-south-1", - NumberOfObjects: 2, - TotalSize: 830, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - RedirectLocation: "/something", - Metadata: map[string]string{}, - }, - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - RedirectLocation: "/something", - Metadata: map[string]string{}, - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -376,99 +526,24 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Error: errors.New("ListObjectsForbidden"), - - ExpectedResp: []*model.MetaBucket{}, - ExpectedError: nil, - }, - - { - TestNumber: 10, - ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ - {CreationDate: &TIMEVAL, Name: aws.String("xyz")}, - }, - }, - GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, - GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectTaggingResp: &s3.GetObjectTaggingOutput{TagSet: []*s3.Tag{{Key: aws.String("key1"), Value: aws.String("val1")}}, VersionId: aws.String("dubwvnkewv.fewvbib")}, - GetObjectAclError: errors.New("GetObjectAclError"), - GetBucketAclError: errors.New("GetBucketAclError"), - - ExpectedResp: []*model.MetaBucket{ - { - CreationDate: &TIMEVAL, - Name: "xyz", - Region: "ap-south-1", - NumberOfObjects: 2, - TotalSize: 830, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - ObjectTags: map[string]string{"key1": "val1"}, - VersionId: "dubwvnkewv.fewvbib", - }, - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - ObjectTags: map[string]string{"key1": "val1"}, - VersionId: "dubwvnkewv.fewvbib", - }, - }, - }, - }, - ExpectedError: nil, - }, - - { - TestNumber: 11, - ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ - {CreationDate: &TIMEVAL, Name: aws.String("abc")}, - }, - }, - GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, - GetBucketTaggingError: errors.New("GetBucketTaggingError"), GetBucketAclResp: &s3.GetBucketAclOutput{Grants: []*s3.Grant{{Grantee: &s3.Grantee{ID: aws.String("ewwiubibu")}, Permission: aws.String("FULL_CONTROL")}}}, - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), - GetObjectAclResp: &s3.GetObjectAclOutput{Grants: []*s3.Grant{{Grantee: &s3.Grantee{ID: aws.String("ewwiubibu")}, Permission: aws.String("FULL_CONTROL")}}}, - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 1, - TotalSize: 415, + NumberOfObjects: 0, + TotalSize: 0, BucketAcl: []*model.Access{{ID: "ewwiubibu", Permission: "FULL_CONTROL"}}, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - ObjectAcl: []*model.Access{{ID: "ewwiubibu", Permission: "FULL_CONTROL"}}, - }, - }, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, }, { - TestNumber: 12, + TestNumber: 10, ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ {CreationDate: &TIMEVAL, Name: aws.String("abc")}, {CreationDate: &TIMEVAL, Name: aws.String("qqq")}, @@ -477,51 +552,31 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 1, - TotalSize: 415, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, { CreationDate: &TIMEVAL, Name: "ghi", Region: "ap-south-1", - NumberOfObjects: 1, - TotalSize: 415, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, }, { - TestNumber: 13, + TestNumber: 11, ListBucketsResp: &s3.ListBucketsOutput{Buckets: []*s3.Bucket{ {CreationDate: &TIMEVAL, Name: aws.String("abc")}, {CreationDate: &TIMEVAL, Name: aws.String("qqq")}, @@ -530,29 +585,16 @@ func TestBucketList(t *testing.T) { }, GetBucketLocationResp: &s3.GetBucketLocationOutput{LocationConstraint: aws.String("ap-south-1")}, GetBucketTaggingError: errors.New("GetBucketTaggingError"), - ListObjectsV2Resp: &s3.ListObjectsV2Output{Contents: []*s3.Object{ - {Key: aws.String("object1"), LastModified: &TIMEVAL, Size: aws.Int64(415), StorageClass: aws.String("STANDARD")}, - }}, - HeadObjectError: errors.New("HeadObjectError"), - GetObjectAclError: errors.New("GetObjectAclError"), - GetObjectTaggingError: errors.New("GetObjectTaggingError"), GetBucketAclError: errors.New("GetBucketAclError"), - ExpectedResp: []*model.MetaBucket{ + ExpectedBucketResp: []*model.MetaBucket{ { CreationDate: &TIMEVAL, Name: "abc", Region: "ap-south-1", - NumberOfObjects: 1, - TotalSize: 415, - Objects: []*model.MetaObject{ - { - ObjectName: "object1", - LastModifiedDate: &TIMEVAL, - Size: 415, - StorageClass: "STANDARD", - }, - }, + NumberOfObjects: 0, + TotalSize: 0, + Objects: []*model.MetaObject{}, }, }, ExpectedError: nil, @@ -579,9 +621,10 @@ func TestBucketList(t *testing.T) { GetObjectTaggingResp: test.GetObjectTaggingResp, GetObjectTaggingError: test.GetObjectTaggingError, } + ObjectList = MockObjectList bucketArray, err := BucketList(mockSvc, &backendRegion) - assert.Equal(t, test.ExpectedResp, bucketArray, test.TestNumber) + assert.Equal(t, test.ExpectedBucketResp, bucketArray, test.TestNumber) assert.Equal(t, test.ExpectedError, err, test.TestNumber) } } From 18da7e79ee66da5b57bce833e785c2c81b0965c2 Mon Sep 17 00:00:00 2001 From: Devansh Jain Date: Tue, 21 Feb 2023 11:59:29 +0530 Subject: [PATCH 3/3] modify a test case in TestObjectList function --- metadata/pkg/drivers/aws/aws_test.go | 43 ++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/metadata/pkg/drivers/aws/aws_test.go b/metadata/pkg/drivers/aws/aws_test.go index a0e80ddb0..3c612ffd6 100644 --- a/metadata/pkg/drivers/aws/aws_test.go +++ b/metadata/pkg/drivers/aws/aws_test.go @@ -30,6 +30,26 @@ import ( func MockObjectList(svc s3iface.S3API, bucketName string) ([]*model.MetaObject, int64, error) { if bucketName == "ppp" { return nil, 0, errors.New("ListObjectsV2Error") + } else if bucketName == "vui" { + objectArray := []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + } + return objectArray, 830, nil } return []*model.MetaObject{}, 0, nil } @@ -510,9 +530,26 @@ func TestBucketList(t *testing.T) { CreationDate: &TIMEVAL, Name: "vui", Region: "ap-south-1", - NumberOfObjects: 0, - TotalSize: 0, - Objects: []*model.MetaObject{}, + NumberOfObjects: 2, + TotalSize: 830, + Objects: []*model.MetaObject{ + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + { + ObjectName: "object1", + LastModifiedDate: &TIMEVAL, + Size: 415, + StorageClass: "STANDARD", + RedirectLocation: "/something", + Metadata: map[string]string{}, + }, + }, }, }, ExpectedError: nil,