From 42305a4628b64b9a6714814b98258291618ba79f Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Thu, 29 Dec 2022 07:58:13 -0500 Subject: [PATCH 1/5] Initiate work on PutObjectTagging ref PR#71 and issue #72 --- awscli_test.go | 20 ++++++------ backend.go | 3 +- backend/s3mem/backend.go | 3 +- backend/s3mem/bucket.go | 2 ++ gofakes3.go | 69 +++++++++++++++++++++++++++++++++++++--- gofakes3_test.go | 55 ++++++++++++++++++++------------ init_test.go | 8 ++--- messages.go | 10 ++++++ routing.go | 23 ++++++++++++-- routing_test.go | 2 +- 10 files changed, 152 insertions(+), 43 deletions(-) diff --git a/awscli_test.go b/awscli_test.go index 6af3d567..d9ff787d 100644 --- a/awscli_test.go +++ b/awscli_test.go @@ -44,20 +44,20 @@ func TestCLILsFiles(t *testing.T) { t.Fatal() } - cli.backendPutString(defaultBucket, "test-one", nil, "hello") + cli.backendPutString(defaultBucket, "test-one", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"test-one"}) - cli.backendPutString(defaultBucket, "test-two", nil, "hello") + cli.backendPutString(defaultBucket, "test-two", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"test-one", "test-two"}) // only "test-one" and "test-two" should pass the prefix match - cli.backendPutString(defaultBucket, "no-match", nil, "hello") + cli.backendPutString(defaultBucket, "no-match", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "test-", nil, []string{"test-one", "test-two"}) - cli.backendPutString(defaultBucket, "test/yep", nil, "hello") + cli.backendPutString(defaultBucket, "test/yep", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", []string{"test/"}, []string{"no-match", "test-one", "test-two"}) @@ -74,8 +74,8 @@ func TestCLIRmOne(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutString(defaultBucket, "foo", nil, "hello") - cli.backendPutString(defaultBucket, "bar", nil, "hello") + cli.backendPutString(defaultBucket, "foo", nil, nil, "hello") + cli.backendPutString(defaultBucket, "bar", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar"}) cli.rm(cli.fileArg(defaultBucket, "foo")) @@ -86,9 +86,9 @@ func TestCLIRmMulti(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutString(defaultBucket, "foo", nil, "hello") - cli.backendPutString(defaultBucket, "bar", nil, "hello") - cli.backendPutString(defaultBucket, "baz", nil, "hello") + cli.backendPutString(defaultBucket, "foo", nil, nil, "hello") + cli.backendPutString(defaultBucket, "bar", nil, nil, "hello") + cli.backendPutString(defaultBucket, "baz", nil, nil, "hello") cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar", "baz"}) cli.rmMulti(defaultBucket, "foo", "bar", "baz") @@ -117,7 +117,7 @@ func TestCLIDownload(t *testing.T) { cli := newTestCLI(t) defer cli.Close() - cli.backendPutBytes(defaultBucket, "foo", nil, tc.in) + cli.backendPutBytes(defaultBucket, "foo", nil, nil, tc.in) out := cli.download(defaultBucket, "foo") if !bytes.Equal(out, tc.in) { t.Fatal() diff --git a/backend.go b/backend.go index 9fa7fb4f..d25c0864 100644 --- a/backend.go +++ b/backend.go @@ -17,6 +17,7 @@ const ( type Object struct { Name string Metadata map[string]string + Tags map[string]string Size int64 Contents io.ReadCloser Hash []byte @@ -227,7 +228,7 @@ type Backend interface { // // The size can be used if the backend needs to read the whole reader; use // gofakes3.ReadAll() for this job rather than ioutil.ReadAll(). - PutObject(bucketName, key string, meta map[string]string, input io.Reader, size int64) (PutObjectResult, error) + PutObject(bucketName, key string, meta map[string]string, tags map[string]string, input io.Reader, size int64) (PutObjectResult, error) DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error) } diff --git a/backend/s3mem/backend.go b/backend/s3mem/backend.go index 716eaa5f..45918051 100644 --- a/backend/s3mem/backend.go +++ b/backend/s3mem/backend.go @@ -217,7 +217,7 @@ func (db *Backend) GetObject(bucketName, objectName string, rangeRequest *gofake return result, nil } -func (db *Backend) PutObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64) (result gofakes3.PutObjectResult, err error) { +func (db *Backend) PutObject(bucketName, objectName string, meta map[string]string, tags map[string]string, input io.Reader, size int64) (result gofakes3.PutObjectResult, err error) { // No need to lock the backend while we read the data into memory; it holds // the write lock open unnecessarily, and could be blocked for an unreasonably // long time by a connection timing out: @@ -247,6 +247,7 @@ func (db *Backend) PutObject(bucketName, objectName string, meta map[string]stri hash: hash[:], etag: `"` + hex.EncodeToString(hash[:]) + `"`, metadata: meta, + tags: tags, lastModified: db.timeSource.Now(), } diff --git a/backend/s3mem/bucket.go b/backend/s3mem/bucket.go index a713d110..1ffea7ef 100644 --- a/backend/s3mem/bucket.go +++ b/backend/s3mem/bucket.go @@ -120,6 +120,7 @@ type bucketData struct { hash []byte etag string metadata map[string]string + tags map[string]string } func (bi *bucketData) toObject(rangeRequest *gofakes3.ObjectRangeRequest, withBody bool) (obj *gofakes3.Object, err error) { @@ -152,6 +153,7 @@ func (bi *bucketData) toObject(rangeRequest *gofakes3.ObjectRangeRequest, withBo Name: bi.name, Hash: bi.hash, Metadata: bi.metadata, + Tags: bi.tags, Size: sz, Range: rnge, IsDeleteMarker: bi.deleteMarker, diff --git a/gofakes3.go b/gofakes3.go index f4ec9fad..74864074 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -549,7 +549,7 @@ func (g *GoFakeS3) createObjectBrowserUpload(bucket string, w http.ResponseWrite return err } - result, err := g.storage.PutObject(bucket, key, meta, rdr, fileHeader.Size) + result, err := g.storage.PutObject(bucket, key, meta, nil, rdr, fileHeader.Size) if err != nil { return err } @@ -623,7 +623,7 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r return err } - result, err := g.storage.PutObject(bucket, object, meta, rdr, size) + result, err := g.storage.PutObject(bucket, object, meta, nil, rdr, size) if err != nil { return err } @@ -681,7 +681,7 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h } } - result, err := g.storage.PutObject(bucket, object, meta, srcObj.Contents, srcObj.Size) + result, err := g.storage.PutObject(bucket, object, meta, nil, srcObj.Contents, srcObj.Size) if err != nil { return err } @@ -881,6 +881,67 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload return nil } +// From the docs: +// +// Updating is usually done via different e.g., PutObject routes; but +// PutObjectTagging is one example where existing objects are updated in situ +// +// Sets the supplied tag-set to an object that already exists in a bucket. +// +// A tag is a key-value pair. You can associate tags with an object by sending a +// PUT request against the tagging subresource that is associated with the +// object. +func (g *GoFakeS3) updateObjectWithTags(bucket, object string, version string, w http.ResponseWriter, r *http.Request) error { + // write keys / metadata to object + if err := g.ensureBucketExists(bucket); err != nil { + return err + } + + var in Tagging + if err := g.xmlDecodeBody(r.Body, &in); err != nil { + return err + } + + rnge, err := parseRangeHeader(r.Header.Get("Range")) + if err != nil { + return err + } + + obj, err := g.storage.GetObject(bucket, object, rnge) + if err != nil { + return err + } + if obj.Tags == nil { + obj.Tags = map[string]string{} + } + for _, v := range in.TagSet.Tag { + obj.Tags[v.Key] = v.Value + } + + result, err := g.storage.PutObject(bucket, object, obj.Metadata, obj.Tags, obj.Contents, obj.Size) + if err != nil { + return err + } + if result.VersionID != "" { + w.Header().Set("x-amz-version-id", string(result.VersionID)) + } + + return nil +} + +func (g *GoFakeS3) getObjectTags(bucket, object string, version string, w http.ResponseWriter, r *http.Request) error { + obj, err := g.storage.HeadObject(bucket, object) + if err != nil { + return err + } + out := Tagging{} + for k, v := range obj.Tags { + out.TagSet.Tag = append(out.TagSet.Tag, Tag{Key: k, Value: v}) + } + + return g.xmlEncoder(w).Encode(out) +} + func (g *GoFakeS3) abortMultipartUpload(bucket, object string, uploadID UploadID, w http.ResponseWriter, r *http.Request) error { g.log.Print(LogInfo, "abort multipart upload", bucket, object, uploadID) if _, err := g.uploader.Complete(bucket, object, uploadID); err != nil { @@ -908,7 +969,7 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - result, err := g.storage.PutObject(bucket, object, upload.Meta, bytes.NewReader(fileBody), int64(len(fileBody))) + result, err := g.storage.PutObject(bucket, object, upload.Meta, nil, bytes.NewReader(fileBody), int64(len(fileBody))) if err != nil { return err } diff --git a/gofakes3_test.go b/gofakes3_test.go index 36205e67..e69481c7 100644 --- a/gofakes3_test.go +++ b/gofakes3_test.go @@ -282,10 +282,11 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { defer ts.Close() svc := ts.s3Client() + expectedBody := "hello" _, err := svc.PutObject(&s3.PutObjectInput{ Bucket: aws.String(defaultBucket), Key: aws.String("object"), - Body: bytes.NewReader([]byte("hello")), + Body: strings.NewReader(expectedBody), Metadata: map[string]*string{ "Test": aws.String("test"), }, @@ -316,7 +317,7 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { Key: aws.String("object"), Tagging: &s3.Tagging{ TagSet: []*s3.Tag{ - {Key: aws.String("Tag-Test"), Value: aws.String("test")}, + {Key: aws.String("TagTest"), Value: aws.String("test")}, }, }, }) @@ -341,9 +342,23 @@ func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { }) ts.OK(err) - if *result.TagSet[0].Key != "Tag-Test" && *result.TagSet[0].Value != "test" { + if *result.TagSet[0].Key != "TagTest" && *result.TagSet[0].Value != "test" { t.Fatalf("tag set wrong: %+v", head.Metadata) } + + // receive object after tagging + get, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(defaultBucket), + Key: aws.String("object"), + }) + ts.OK(err) + + c, err := io.ReadAll(get.Body) + ts.OK(err) + + if string(c) != expectedBody { + t.Fatalf("body has been changed: %s", string(c)) + } } func TestCopyObject(t *testing.T) { @@ -356,7 +371,7 @@ func TestCopyObject(t *testing.T) { "X-Amz-Meta-One": "src", "X-Amz-Meta-Two": "src", } - ts.backendPutString(defaultBucket, "src-key", srcMeta, "content") + ts.backendPutString(defaultBucket, "src-key", srcMeta, nil, "content") out, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -407,7 +422,7 @@ func TestCopyObjectWithSpecialChars(t *testing.T) { } srcKey := "src+key,with special;chars!?=" content := "contents" - ts.backendPutString(defaultBucket, srcKey, srcMeta, content) + ts.backendPutString(defaultBucket, srcKey, srcMeta, nil, content) copySource := "/" + defaultBucket + "/" + url.QueryEscape(srcKey) _, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -440,7 +455,7 @@ func TestCopyObjectWithSpecialCharsEscapedInvalied(t *testing.T) { } srcKey := "src+key" //encoded srcKey = src%2Bkey content := "contents" - ts.backendPutString(defaultBucket, srcKey, srcMeta, content) + ts.backendPutString(defaultBucket, srcKey, srcMeta, nil, content) copySource := "/" + defaultBucket + "/src%2key" //invalid encoding _, err := svc.CopyObject(&s3.CopyObjectInput{ Bucket: aws.String(defaultBucket), @@ -470,7 +485,7 @@ func TestDeleteBucket(t *testing.T) { svc := ts.s3Client() ts.backendCreateBucket("test") - ts.backendPutString("test", "test", nil, "test") + ts.backendPutString("test", "test", nil, nil, "test") _, err := svc.DeleteBucket(&s3.DeleteBucketInput{ Bucket: aws.String("test"), }) @@ -503,9 +518,9 @@ func TestDeleteMulti(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") - ts.backendPutString(defaultBucket, "bar", nil, "two") - ts.backendPutString(defaultBucket, "baz", nil, "three") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") + ts.backendPutString(defaultBucket, "bar", nil, nil, "two") + ts.backendPutString(defaultBucket, "baz", nil, nil, "three") rs, err := svc.DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(defaultBucket), @@ -525,9 +540,9 @@ func TestDeleteMulti(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") - ts.backendPutString(defaultBucket, "bar", nil, "two") - ts.backendPutString(defaultBucket, "baz", nil, "three") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") + ts.backendPutString(defaultBucket, "bar", nil, nil, "two") + ts.backendPutString(defaultBucket, "baz", nil, nil, "three") rs, err := svc.DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(defaultBucket), @@ -549,7 +564,7 @@ func TestGetBucketLocation(t *testing.T) { defer ts.Close() svc := ts.s3Client() - ts.backendPutString(defaultBucket, "foo", nil, "one") + ts.backendPutString(defaultBucket, "foo", nil, nil, "one") out, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{ Bucket: aws.String(defaultBucket), @@ -616,7 +631,7 @@ func TestGetObjectRange(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutBytes(defaultBucket, "foo", nil, in) + ts.backendPutBytes(defaultBucket, "foo", nil, nil, in) assertRange(ts, "foo", tc.hdr, tc.expected, tc.fail) }) } @@ -647,7 +662,7 @@ func TestGetObjectRangeInvalid(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutBytes(defaultBucket, "foo", nil, in) + ts.backendPutBytes(defaultBucket, "foo", nil, nil, in) assertRangeInvalid(ts, "foo", tc.hdr) }) } @@ -692,7 +707,7 @@ func TestGetObjectIfNoneMatch(t *testing.T) { ts := newTestServer(t) defer ts.Close() - ts.backendPutString(defaultBucket, objectKey, nil, "hello") + ts.backendPutString(defaultBucket, objectKey, nil, nil, "hello") assertModified(ts, tc.ifNoneMatch, tc.shouldModify) }) @@ -1012,7 +1027,7 @@ func TestObjectVersions(t *testing.T) { const neverVerBucket = "neverver" ts.backendCreateBucket(neverVerBucket) - ts.backendPutString(neverVerBucket, "object", nil, "body 1") + ts.backendPutString(neverVerBucket, "object", nil, nil, "body 1") list(ts, neverVerBucket, "null") // S300005 }) } @@ -1022,7 +1037,7 @@ func TestListBucketPages(t *testing.T) { keys := make([]string, n) for i := int64(0); i < n; i++ { key := fmt.Sprintf("%s%d", prefix, i) - ts.backendPutString(defaultBucket, key, nil, fmt.Sprintf("body-%d", i)) + ts.backendPutString(defaultBucket, key, nil, nil, fmt.Sprintf("body-%d", i)) keys[i] = key } return keys @@ -1127,7 +1142,7 @@ func TestListBucketPagesFallback(t *testing.T) { keys := make([]string, n) for i := int64(0); i < n; i++ { key := fmt.Sprintf("%s%d", prefix, i) - ts.backendPutString(defaultBucket, key, nil, fmt.Sprintf("body-%d", i)) + ts.backendPutString(defaultBucket, key, nil, nil, fmt.Sprintf("body-%d", i)) keys[i] = key } return keys diff --git a/init_test.go b/init_test.go index 0fb3a5b5..0bbe2914 100644 --- a/init_test.go +++ b/init_test.go @@ -248,14 +248,14 @@ func (ts *testServer) backendObjectExists(bucket, key string) bool { return obj != nil } -func (ts *testServer) backendPutString(bucket, key string, meta map[string]string, in string) { +func (ts *testServer) backendPutString(bucket, key string, meta map[string]string, tags map[string]string, in string) { ts.Helper() - ts.OKAll(ts.backend.PutObject(bucket, key, meta, strings.NewReader(in), int64(len(in)))) + ts.OKAll(ts.backend.PutObject(bucket, key, meta, tags, strings.NewReader(in), int64(len(in)))) } -func (ts *testServer) backendPutBytes(bucket, key string, meta map[string]string, in []byte) { +func (ts *testServer) backendPutBytes(bucket, key string, meta map[string]string, tags map[string]string, in []byte) { ts.Helper() - ts.OKAll(ts.backend.PutObject(bucket, key, meta, bytes.NewReader(in), int64(len(in)))) + ts.OKAll(ts.backend.PutObject(bucket, key, meta, tags, bytes.NewReader(in), int64(len(in)))) } func (ts *testServer) backendGetString(bucket, key string, rnge *gofakes3.ObjectRangeRequest) string { diff --git a/messages.go b/messages.go index 17138fe5..533aa895 100644 --- a/messages.go +++ b/messages.go @@ -55,6 +55,16 @@ type CompletedPart struct { type CompleteMultipartUploadRequest struct { Parts []CompletedPart `xml:"Part"` } +type Tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} +type Tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet struct { + Tag []Tag `xml:"Tag"` + } `xml:"TagSet"` +} func (c CompleteMultipartUploadRequest) partsAreSorted() bool { return sort.IntsAreSorted(c.partIDs()) diff --git a/routing.go b/routing.go index e139a15b..e293e48a 100644 --- a/routing.go +++ b/routing.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "net/http" + "net/url" "strings" ) @@ -12,12 +13,12 @@ import ( // // URLs are assumed to break down into two common path segments, in the // following format: -// // +// +// // // // The operation for most of the core functionality is built around HTTP // verbs, but outside the core functionality, the clean separation starts // to degrade, especially around multipart uploads. -// func (g *GoFakeS3) routeBase(w http.ResponseWriter, r *http.Request) { var ( path = strings.Trim(r.URL.Path, "/") @@ -48,6 +49,9 @@ func (g *GoFakeS3) routeBase(w http.ResponseWriter, r *http.Request) { } else if _, ok := query["versioning"]; ok { err = g.routeVersioning(bucket, w, r) + } else if _, ok := query["tagging"]; ok { + err = g.routeTagging(bucket, object, query, w, r) + } else if _, ok := query["versions"]; ok { err = g.routeVersions(bucket, w, r) @@ -155,6 +159,19 @@ func (g *GoFakeS3) routeVersions(bucket string, w http.ResponseWriter, r *http.R } } +// routeTagging operates on routes that contain '?tagging' in the query string. +func (g *GoFakeS3) routeTagging(bucket, object string, query url.Values, w http.ResponseWriter, r *http.Request) error { + versionID := versionFromQuery(query["versionId"]) + switch r.Method { + case "PUT": + return g.updateObjectWithTags(bucket, object, versionID, w, r) + case "GET": + return g.getObjectTags(bucket, object, versionID, w, r) + default: + return ErrMethodNotAllowed + } +} + // routeVersion operates on routes that contain '?versionId=' in the // query string. func (g *GoFakeS3) routeVersion(bucket, object string, versionID VersionID, w http.ResponseWriter, r *http.Request) error { @@ -165,6 +182,8 @@ func (g *GoFakeS3) routeVersion(bucket, object string, versionID VersionID, w ht return g.headObject(bucket, object, versionID, w, r) case "DELETE": return g.deleteObjectVersion(bucket, object, versionID, w, r) + case "PUT": + return g.updateObjectWithTags(bucket, object, string(versionID), w, r) default: return ErrMethodNotAllowed } diff --git a/routing_test.go b/routing_test.go index cc427da5..88a07abc 100644 --- a/routing_test.go +++ b/routing_test.go @@ -8,7 +8,7 @@ func TestRoutingSlashes(t *testing.T) { ts := newTestServer(t, withoutInitialBuckets()) defer ts.Close() ts.backendCreateBucket("test") - ts.backendPutString("test", "obj", nil, "yep") + ts.backendPutString("test", "obj", nil, nil, "yep") client := httpClient() From 939fd63444a13c9a55af97e98f0bf6bcee16c69b Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Thu, 29 Dec 2022 08:10:07 -0500 Subject: [PATCH 2/5] Add object tags to bolt backend --- backend/s3bolt/backend.go | 2 ++ backend/s3bolt/schema.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/backend/s3bolt/backend.go b/backend/s3bolt/backend.go index 5dda0e40..5e73d13a 100644 --- a/backend/s3bolt/backend.go +++ b/backend/s3bolt/backend.go @@ -294,6 +294,7 @@ func (db *Backend) GetObject(bucketName, objectName string, rangeRequest *gofake func (db *Backend) PutObject( bucketName, objectName string, meta map[string]string, + tags map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { @@ -319,6 +320,7 @@ func (db *Backend) PutObject( data, err := bson.Marshal(&boltObject{ Name: objectName, Metadata: meta, + Tags: tags, Size: int64(len(bts)), LastModified: mod, Contents: bts, diff --git a/backend/s3bolt/schema.go b/backend/s3bolt/schema.go index 44baff55..e7f8013a 100644 --- a/backend/s3bolt/schema.go +++ b/backend/s3bolt/schema.go @@ -23,6 +23,7 @@ type boltBucket struct { type boltObject struct { Name string Metadata map[string]string + Tags map[string]string LastModified time.Time Size int64 Contents []byte @@ -44,6 +45,7 @@ func (b *boltObject) Object(objectName string, rangeRequest *gofakes3.ObjectRang return &gofakes3.Object{ Name: objectName, Metadata: b.Metadata, + Tags: b.Tags, Size: b.Size, Contents: s3io.ReaderWithDummyCloser{bytes.NewReader(data)}, Range: rnge, From 992660b8fe316f08cc9c5927d9815eb95a22bdb0 Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Thu, 29 Dec 2022 08:23:34 -0500 Subject: [PATCH 3/5] Add object tagging to s3afero --- backend/s3afero/meta.go | 1 + backend/s3afero/multi.go | 6 +++++- backend/s3afero/single.go | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/s3afero/meta.go b/backend/s3afero/meta.go index ec16a0eb..ab08af98 100644 --- a/backend/s3afero/meta.go +++ b/backend/s3afero/meta.go @@ -15,6 +15,7 @@ import ( type Metadata struct { File string ModTime time.Time + Tags map[string]string Size int64 Hash []byte Meta map[string]string diff --git a/backend/s3afero/multi.go b/backend/s3afero/multi.go index 9f758c9f..b68d9574 100644 --- a/backend/s3afero/multi.go +++ b/backend/s3afero/multi.go @@ -25,7 +25,6 @@ import ( // It is STRONGLY recommended that the metadata Fs is not contained within the // `/buckets` subdirectory as that could make a significant mess, but this is // infeasible to validate, so you're encouraged to be extremely careful! -// type MultiBucketBackend struct { lock sync.Mutex baseFs afero.Fs @@ -306,6 +305,7 @@ func (db *MultiBucketBackend) HeadObject(bucketName, objectName string) (*gofake Name: objectName, Hash: meta.Hash, Metadata: meta.Meta, + Tags: meta.Tags, Size: size, Contents: s3io.NoOpReadCloser{}, }, nil @@ -368,6 +368,7 @@ func (db *MultiBucketBackend) GetObject(bucketName, objectName string, rangeRequ Name: objectName, Hash: meta.Hash, Metadata: meta.Meta, + Tags: meta.Tags, Range: rnge, Size: size, Contents: rdr, @@ -377,9 +378,11 @@ func (db *MultiBucketBackend) GetObject(bucketName, objectName string, rangeRequ func (db *MultiBucketBackend) PutObject( bucketName, objectName string, meta map[string]string, + tags map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { + // merge metadata err = gofakes3.MergeMetadata(db, bucketName, objectName, meta) if err != nil { return result, err @@ -441,6 +444,7 @@ func (db *MultiBucketBackend) PutObject( storedMeta := &Metadata{ File: objectPath, Hash: hasher.Sum(nil), + Tags: tags, Meta: meta, Size: stat.Size(), ModTime: stat.ModTime(), diff --git a/backend/s3afero/single.go b/backend/s3afero/single.go index e671181e..1af5ba4e 100644 --- a/backend/s3afero/single.go +++ b/backend/s3afero/single.go @@ -28,7 +28,6 @@ import ( // It is STRONGLY recommended that the metadata Fs is not contained within the // `/buckets` subdirectory as that could make a significant mess, but this is // infeasible to validate, so you're encouraged to be extremely careful! -// type SingleBucketBackend struct { lock sync.Mutex fs afero.Fs @@ -223,6 +222,7 @@ func (db *SingleBucketBackend) HeadObject(bucketName, objectName string) (*gofak Name: objectName, Hash: meta.Hash, Metadata: meta.Meta, + Tags: meta.Tags, Size: size, Contents: s3io.NoOpReadCloser{}, }, nil @@ -288,6 +288,7 @@ func (db *SingleBucketBackend) GetObject(bucketName, objectName string, rangeReq func (db *SingleBucketBackend) PutObject( bucketName, objectName string, meta map[string]string, + tags map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { @@ -349,6 +350,7 @@ func (db *SingleBucketBackend) PutObject( File: objectName, Hash: hasher.Sum(nil), Meta: meta, + Tags: tags, Size: stat.Size(), ModTime: stat.ModTime(), } From 4d9f6f20aa90abab08f2fd8be7bb1d11796be94a Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Thu, 29 Dec 2022 08:27:49 -0500 Subject: [PATCH 4/5] Upgrade sfafero tests to comply with changed PutObject signature --- backend/s3afero/backend_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/s3afero/backend_test.go b/backend/s3afero/backend_test.go index cb674c32..cc84f145 100644 --- a/backend/s3afero/backend_test.go +++ b/backend/s3afero/backend_test.go @@ -43,7 +43,7 @@ func TestPutGet(t *testing.T) { } contents := []byte("contents") - if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil { + if _, err := backend.PutObject("test", "yep", meta, nil, bytes.NewReader(contents), int64(len(contents))); err != nil { t.Fatal(err) } hasher := md5.New() @@ -89,7 +89,7 @@ func TestPutGetRange(t *testing.T) { contents := []byte("contents") expected := contents[1:7] - if _, err := backend.PutObject("test", "yep", meta, bytes.NewReader(contents), int64(len(contents))); err != nil { + if _, err := backend.PutObject("test", "yep", meta, nil, bytes.NewReader(contents), int64(len(contents))); err != nil { t.Fatal(err) } hasher := md5.New() @@ -134,12 +134,12 @@ func TestPutListRoot(t *testing.T) { } contents1 := []byte("contents1") - if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil { + if _, err := backend.PutObject("test", "foo", meta, nil, bytes.NewReader(contents1), int64(len(contents1))); err != nil { t.Fatal(err) } contents2 := []byte("contents2") - if _, err := backend.PutObject("test", "bar", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil { + if _, err := backend.PutObject("test", "bar", meta, nil, bytes.NewReader(contents2), int64(len(contents2))); err != nil { t.Fatal(err) } @@ -180,12 +180,12 @@ func TestPutListDir(t *testing.T) { } contents1 := []byte("contents1") - if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil { + if _, err := backend.PutObject("test", "foo/bar", meta, nil, bytes.NewReader(contents1), int64(len(contents1))); err != nil { t.Fatal(err) } contents2 := []byte("contents2") - if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil { + if _, err := backend.PutObject("test", "foo/baz", meta, nil, bytes.NewReader(contents2), int64(len(contents2))); err != nil { t.Fatal(err) } @@ -226,7 +226,7 @@ func TestPutDelete(t *testing.T) { } contents := []byte("contents1") - if _, err := backend.PutObject("test", "foo", meta, bytes.NewReader(contents), int64(len(contents))); err != nil { + if _, err := backend.PutObject("test", "foo", meta, nil, bytes.NewReader(contents), int64(len(contents))); err != nil { t.Fatal(err) } @@ -258,12 +258,12 @@ func TestPutDeleteMulti(t *testing.T) { } contents1 := []byte("contents1") - if _, err := backend.PutObject("test", "foo/bar", meta, bytes.NewReader(contents1), int64(len(contents1))); err != nil { + if _, err := backend.PutObject("test", "foo/bar", meta, nil, bytes.NewReader(contents1), int64(len(contents1))); err != nil { t.Fatal(err) } contents2 := []byte("contents2") - if _, err := backend.PutObject("test", "foo/baz", meta, bytes.NewReader(contents2), int64(len(contents2))); err != nil { + if _, err := backend.PutObject("test", "foo/baz", meta, nil, bytes.NewReader(contents2), int64(len(contents2))); err != nil { t.Fatal(err) } From 940f235ec3d4af4ba6087d3b92551bd84d20e05e Mon Sep 17 00:00:00 2001 From: Johannes Boyne Date: Sat, 17 Feb 2024 11:37:18 +0100 Subject: [PATCH 5/5] Fix test cases for s3afero --- backend.go | 2 +- backend/s3afero/single.go | 1 + uploader.go | 15 ++++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend.go b/backend.go index 31683333..aee5e051 100644 --- a/backend.go +++ b/backend.go @@ -340,7 +340,7 @@ func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta ma } defer c.Contents.Close() - _, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + _, err = db.PutObject(dstBucket, dstKey, meta, c.Tags, c.Contents, c.Size) if err != nil { return } diff --git a/backend/s3afero/single.go b/backend/s3afero/single.go index 2aca0f29..e2056eec 100644 --- a/backend/s3afero/single.go +++ b/backend/s3afero/single.go @@ -216,6 +216,7 @@ func (db *SingleBucketBackend) ensureMeta( return &Metadata{ objectPath, mtime, + nil, size, hash, map[string]string{}, diff --git a/uploader.go b/uploader.go index f5f92ec6..e45021c0 100644 --- a/uploader.go +++ b/uploader.go @@ -28,6 +28,7 @@ A skiplist that maps object keys to upload ids is also maintained to support the ListMultipartUploads operation. From the docs: + In the response, the uploads are sorted by key. If your application has initiated more than one multipart upload using the same object key, then uploads in the response are first sorted by key. Additionally, @@ -135,13 +136,13 @@ func (bu *bucketUploads) remove(uploadID UploadID) { // Multipart upload support has the following rather severe limitations (which // will hopefully be addressed in the future): // -// - uploads do not interface with the Backend, so they do not -// currently persist across reboots +// - uploads do not interface with the Backend, so they do not +// currently persist across reboots // -// - upload parts are held in memory, so if you want to upload something huge -// in multiple parts (which is pretty much exactly what you'd want multipart -// uploads for), you'll need to make sure your memory is also sufficiently -// huge! +// - upload parts are held in memory, so if you want to upload something huge +// in multiple parts (which is pretty much exactly what you'd want multipart +// uploads for), you'll need to make sure your memory is also sufficiently +// huge! // // At this stage, the current thinking would be to add a second optional // Backend interface that allows persistent operations on multipart upload @@ -448,7 +449,7 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i hash := fmt.Sprintf("%x", md5.Sum(body)) - result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body))) + result, err := u.storage.PutObject(bucket, object, mpu.Meta, nil, bytes.NewReader(body), int64(len(body))) if err != nil { return "", "", err }