diff --git a/README.md b/README.md index 2ee4cc81..a1bb6040 100644 --- a/README.md +++ b/README.md @@ -199,4 +199,4 @@ A big thank you to all the [contributors](https://github.com/johannesboyne/gofak especially [Blake @shabbyrobe](https://github.com/shabbyrobe) who pushed this little project to the next level! -**Help wanred** +**Help wanted** diff --git a/backend.go b/backend.go index d25c0864..31683333 100644 --- a/backend.go +++ b/backend.go @@ -1,7 +1,9 @@ package gofakes3 import ( + "encoding/hex" "io" + "time" "github.com/aws/aws-sdk-go/aws/awserr" ) @@ -231,6 +233,8 @@ type Backend interface { PutObject(bucketName, key string, meta map[string]string, tags map[string]string, input io.Reader, size int64) (PutObjectResult, error) DeleteMulti(bucketName string, objects ...string) (MultiDeleteResult, error) + + CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (CopyObjectResult, error) } // VersionedBackend may be optionally implemented by a Backend in order to support @@ -310,6 +314,43 @@ type VersionedBackend interface { ListBucketVersions(bucketName string, prefix *Prefix, page *ListBucketVersionsPage) (*ListBucketVersionsResult, error) } +// MultipartBackend may be optionally implemented by a Backend in order to +// support S3 multiplart uploads. +// If you don't implement MultipartBackend, GoFakeS3 will fall back to an +// in-memory implementation which holds all parts in memory until the upload +// gets finalised and pushed to the backend. +type MultipartBackend interface { + CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error) + UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) + + ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) + ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) + + AbortMultipartUpload(bucket, object string, id UploadID) error + CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error) +} + +// CopyObject is a helper function useful for quickly implementing CopyObject on +// a backend that already supports GetObject and PutObject. This isn't very +// efficient so only use this if performance isn't important. +func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result CopyObjectResult, err error) { + c, err := db.GetObject(srcBucket, srcKey, nil) + if err != nil { + return + } + defer c.Contents.Close() + + _, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + if err != nil { + return + } + + return CopyObjectResult{ + ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + LastModified: NewContentTime(time.Now()), + }, nil +} + func MergeMetadata(db Backend, bucketName string, objectName string, meta map[string]string) error { // get potential existing object to potentially carry metadata over existingObj, err := db.GetObject(bucketName, objectName, nil) diff --git a/backend/s3afero/backend_test.go b/backend/s3afero/backend_test.go index cc84f145..12fab59e 100644 --- a/backend/s3afero/backend_test.go +++ b/backend/s3afero/backend_test.go @@ -298,7 +298,7 @@ func TestMultiCreateBucket(t *testing.T) { } defer os.RemoveAll(tmp) - fs, err := FsPath(tmp) + fs, err := FsPath(tmp, 0) if err != nil { t.Fatal(err) } diff --git a/backend/s3afero/multi.go b/backend/s3afero/multi.go index b68d9574..89c26739 100644 --- a/backend/s3afero/multi.go +++ b/backend/s3afero/multi.go @@ -31,6 +31,7 @@ type MultiBucketBackend struct { bucketFs afero.Fs metaStore *metaStore dirMode os.FileMode + flags FsFlags // FIXME(bw): values in here should not be used beyond the configuration // step; maybe this can be cleaned up later using a builder struct or @@ -47,19 +48,28 @@ func MultiBucket(fs afero.Fs, opts ...MultiOption) (*MultiBucketBackend, error) return nil, err } - b := &MultiBucketBackend{ - baseFs: fs, - bucketFs: afero.NewBasePathFs(fs, "buckets"), - dirMode: 0700, - } + b := &MultiBucketBackend{} for _, opt := range opts { if err := opt(b); err != nil { return nil, err } } + bucketsFs, err := NewBasePathFs(fs, "buckets", FsPathCreateAll) + if err != nil { + return nil, err + } + + b.baseFs = fs + b.bucketFs = bucketsFs + b.dirMode = 0700 + if b.configOnly.metaFs == nil { - b.configOnly.metaFs = afero.NewBasePathFs(fs, "metadata") + metaFs, err := NewBasePathFs(fs, "metadata", FsPathCreateAll) + if err != nil { + return nil, err + } + b.configOnly.metaFs = metaFs } b.metaStore = newMetaStore(b.configOnly.metaFs, modTimeFsCalc(fs)) @@ -141,7 +151,7 @@ func (db *MultiBucketBackend) getBucketWithFilePrefixLocked(bucket string, prefi } if entry.IsDir() { - response.AddPrefix(path.Join(prefixPath, prefixPart)) + response.AddPrefix(path.Join(prefixPath, prefixPart, entry.Name()) + "/") } else { size := entry.Size() @@ -292,6 +302,8 @@ func (db *MultiBucketBackend) HeadObject(bucketName, objectName string) (*gofake return nil, gofakes3.KeyNotFound(objectName) } else if err != nil { return nil, err + } else if stat.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) } size, mtime := stat.Size(), stat.ModTime() @@ -331,7 +343,6 @@ func (db *MultiBucketBackend) GetObject(bucketName, objectName string, rangeRequ } else if err != nil { return nil, err } - defer func() { // If an error occurs, the caller may not have access to Object.Body in order to close it: if obj == nil && rerr != nil { @@ -342,6 +353,8 @@ func (db *MultiBucketBackend) GetObject(bucketName, objectName string, rangeRequ stat, err := f.Stat() if err != nil { return nil, err + } else if stat.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) } size, mtime := stat.Size(), stat.ModTime() @@ -456,6 +469,10 @@ func (db *MultiBucketBackend) PutObject( return result, nil } +func (db *MultiBucketBackend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + return gofakes3.CopyObject(db, srcBucket, srcKey, dstBucket, dstKey, meta) +} + func (db *MultiBucketBackend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { db.lock.Lock() defer db.lock.Unlock() diff --git a/backend/s3afero/option.go b/backend/s3afero/option.go index 83bb863d..2fd1d0eb 100644 --- a/backend/s3afero/option.go +++ b/backend/s3afero/option.go @@ -16,4 +16,11 @@ func MultiWithMetaFs(fs afero.Fs) MultiOption { } } +func MultiFsFlags(flags FsFlags) MultiOption { + return func(b *MultiBucketBackend) error { + b.flags = flags + return nil + } +} + type SingleOption func(b *SingleBucketBackend) error diff --git a/backend/s3afero/single.go b/backend/s3afero/single.go index 1af5ba4e..2aca0f29 100644 --- a/backend/s3afero/single.go +++ b/backend/s3afero/single.go @@ -3,7 +3,7 @@ package s3afero import ( "crypto/md5" "encoding/hex" - "fmt" + "errors" "io" "log" "os" @@ -131,13 +131,12 @@ func (db *SingleBucketBackend) getBucketWithFilePrefixLocked(bucket string, pref } if entry.IsDir() { - response.AddPrefix(path.Join(prefixPath, prefixPart)) + response.AddPrefix(path.Join(prefixPath, entry.Name()) + "/") } else { size := entry.Size() mtime := entry.ModTime() - - meta, err := db.metaStore.loadMeta(bucket, objectPath, size, mtime) + meta, err := db.ensureMeta(bucket, objectPath, size, mtime) if err != nil { return nil, err } @@ -157,31 +156,25 @@ func (db *SingleBucketBackend) getBucketWithFilePrefixLocked(bucket string, pref func (db *SingleBucketBackend) getBucketWithArbitraryPrefixLocked(bucket string, prefix *gofakes3.Prefix) (*gofakes3.ObjectList, error) { response := gofakes3.NewObjectList() - if err := afero.Walk(db.fs, filepath.FromSlash(bucket), func(path string, info os.FileInfo, err error) error { + if err := afero.Walk(db.fs, filepath.FromSlash("."), func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } objectPath := filepath.ToSlash(path) - parts := strings.SplitN(objectPath, "/", 2) - if len(parts) != 2 { - panic(fmt.Errorf("unexpected path %q", path)) // should never happen - } - objectName := parts[1] - - if !prefix.Match(objectName, nil) { + if !prefix.Match(objectPath, nil) { return nil } size := info.Size() mtime := info.ModTime() - meta, err := db.metaStore.loadMeta(bucket, objectName, size, mtime) + meta, err := db.ensureMeta(bucket, objectPath, size, mtime) if err != nil { return err } response.Add(&gofakes3.Content{ - Key: objectName, + Key: objectPath, LastModified: gofakes3.NewContentTime(mtime), ETag: `"` + hex.EncodeToString(meta.Hash) + `"`, Size: size, @@ -196,6 +189,46 @@ func (db *SingleBucketBackend) getBucketWithArbitraryPrefixLocked(bucket string, return response, nil } +func (db *SingleBucketBackend) ensureMeta( + bucket string, + objectPath string, + size int64, + mtime time.Time, +) (meta *Metadata, err error) { + existingMeta, err := db.metaStore.loadMeta(bucket, objectPath, size, mtime) + if errors.Is(err, os.ErrNotExist) { + f, err := db.fs.Open(filepath.FromSlash(objectPath)) + if err != nil { + return nil, err + } + defer f.Close() + + hasher := md5.New() + if _, err := io.Copy(hasher, f); err != nil { + return nil, err + } + + hash, err := hasher.Sum(nil), nil + if err != nil { + return nil, err + } + + return &Metadata{ + objectPath, + mtime, + size, + hash, + map[string]string{}, + }, nil + + } else if err != nil { + return nil, err + + } else { + return existingMeta, nil + } +} + func (db *SingleBucketBackend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) { if bucketName != db.name { return nil, gofakes3.BucketNotFound(bucketName) @@ -209,11 +242,12 @@ func (db *SingleBucketBackend) HeadObject(bucketName, objectName string) (*gofak return nil, gofakes3.KeyNotFound(objectName) } else if err != nil { return nil, err + } else if stat.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) } size, mtime := stat.Size(), stat.ModTime() - - meta, err := db.metaStore.loadMeta(bucketName, objectName, size, mtime) + meta, err := db.ensureMeta(bucketName, objectName, size, mtime) if err != nil { return nil, err } @@ -242,7 +276,6 @@ func (db *SingleBucketBackend) GetObject(bucketName, objectName string, rangeReq } else if err != nil { return nil, err } - defer func() { // If an error occurs, the caller may not have access to Object.Body in order to close it: if err != nil && obj == nil { @@ -253,6 +286,8 @@ func (db *SingleBucketBackend) GetObject(bucketName, objectName string, rangeReq stat, err := f.Stat() if err != nil { return nil, err + } else if stat.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) } size, mtime := stat.Size(), stat.ModTime() @@ -270,7 +305,7 @@ func (db *SingleBucketBackend) GetObject(bucketName, objectName string, rangeReq rdr = limitReadCloser(rdr, f.Close, rnge.Length) } - meta, err := db.metaStore.loadMeta(bucketName, objectName, size, mtime) + meta, err := db.ensureMeta(bucketName, objectName, size, mtime) if err != nil { return nil, err } @@ -387,6 +422,10 @@ func (db *SingleBucketBackend) DeleteMulti(bucketName string, objects ...string) return result, nil } +func (db *SingleBucketBackend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + return gofakes3.CopyObject(db, srcBucket, srcKey, dstBucket, dstKey, meta) +} + func (db *SingleBucketBackend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { if bucketName != db.name { return result, gofakes3.BucketNotFound(bucketName) diff --git a/backend/s3afero/util.go b/backend/s3afero/util.go index c3f349f9..b733869d 100644 --- a/backend/s3afero/util.go +++ b/backend/s3afero/util.go @@ -1,6 +1,7 @@ package s3afero import ( + "errors" "fmt" "io" "os" @@ -44,9 +45,25 @@ func ensureNoOsFs(name string, fs afero.Fs) error { return nil } +func NewBasePathFs(source afero.Fs, path string, flags FsFlags) (afero.Fs, error) { + if flags&(FsPathCreateAll|FsPathCreate) != 0 { + if err := source.MkdirAll(path, 0700); err != nil { + return nil, err + } + } + return afero.NewBasePathFs(source, path), nil +} + +type FsFlags int + +const ( + FsPathCreate FsFlags = 1 << iota + FsPathCreateAll +) + // FsPath returns an afero.Fs rooted to the path provided. If the path is invalid, // or is less than 2 levels down from the filesystem root, an error is returned. -func FsPath(path string) (afero.Fs, error) { +func FsPath(path string, flags FsFlags) (afero.Fs, error) { if path == "" { return nil, fmt.Errorf("gofakes3: empty path") } @@ -57,15 +74,31 @@ func FsPath(path string) (afero.Fs, error) { } stat, err := os.Stat(path) - if err != nil { + if errors.Is(err, os.ErrNotExist) { + if flags&FsPathCreate != 0 { + if err := os.Mkdir(path, 0700); err != nil { + return nil, err + } + } else if flags&FsPathCreateAll != 0 { + if err := os.MkdirAll(path, 0700); err != nil { + return nil, err + } + } else { + return nil, err + } + + } else if err != nil { return nil, err } else if !stat.IsDir() { return nil, fmt.Errorf("gofakes3: path %q is not a directory", path) } parts := strings.Split(path, string(filepath.Separator)) - if len(parts) < 2 { // cheap and nasty footgun check: - return nil, fmt.Errorf("gofakes3: invalid path %q", path) + + // cheap and nasty footgun check to ensure root path is not used + // FIXME: possibly not enough on windows + if len(parts) <= 1 { + return nil, fmt.Errorf("gofakes3: path %q at the root of the file system not allowed; use FsAllowAll to bypass", path) } return afero.NewBasePathFs(afero.NewOsFs(), path), nil diff --git a/backend/s3afero/util_test.go b/backend/s3afero/util_test.go new file mode 100644 index 00000000..b85891a8 --- /dev/null +++ b/backend/s3afero/util_test.go @@ -0,0 +1,44 @@ +package s3afero + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestFsPathCreate(t *testing.T) { + t.Run("default-fails", func(t *testing.T) { + d := filepath.Join(t.TempDir(), "fs") + if _, err := FsPath(d, 0); !errors.Is(err, os.ErrNotExist) { + t.Fatal("expected not exist error, found", err) + } + }) + + t.Run("create", func(t *testing.T) { + d := filepath.Join(t.TempDir(), "fs") + if _, err := FsPath(d, FsPathCreate); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(d); err != nil { + t.Fatal(err) + } + }) + + t.Run("create-nested-fails", func(t *testing.T) { + d := filepath.Join(t.TempDir(), "fs", "nup") + if _, err := FsPath(d, 0); !errors.Is(err, os.ErrNotExist) { + t.Fatal("expected not exist error, found", err) + } + }) + + t.Run("create-all", func(t *testing.T) { + d := filepath.Join(t.TempDir(), "fs", "yep") + if _, err := FsPath(d, FsPathCreateAll); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(d); err != nil { + t.Fatal(err) + } + }) +} diff --git a/backend/s3bolt/backend.go b/backend/s3bolt/backend.go index 5e73d13a..3daf617d 100644 --- a/backend/s3bolt/backend.go +++ b/backend/s3bolt/backend.go @@ -336,6 +336,10 @@ func (db *Backend) PutObject( }) } +func (db *Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + return gofakes3.CopyObject(db, srcBucket, srcKey, dstBucket, dstKey, meta) +} + func (db *Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { return result, db.bolt.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(bucketName)) diff --git a/backend/s3bolt/schema.go b/backend/s3bolt/schema.go index e7f8013a..f329d3bc 100644 --- a/backend/s3bolt/schema.go +++ b/backend/s3bolt/schema.go @@ -47,7 +47,7 @@ func (b *boltObject) Object(objectName string, rangeRequest *gofakes3.ObjectRang Metadata: b.Metadata, Tags: b.Tags, Size: b.Size, - Contents: s3io.ReaderWithDummyCloser{bytes.NewReader(data)}, + Contents: s3io.ReaderWithDummyCloser{Reader: bytes.NewReader(data)}, Range: rnge, Hash: b.Hash, }, nil diff --git a/backend/s3mem/backend.go b/backend/s3mem/backend.go index 45918051..e845013c 100644 --- a/backend/s3mem/backend.go +++ b/backend/s3mem/backend.go @@ -92,7 +92,10 @@ func (db *Backend) ListBucket(name string, prefix *gofakes3.Prefix, page gofakes if page.Marker != "" { iter.Seek(page.Marker) - iter.Next() // Move to the next item after the Marker + // If the current item is the Marker, move to the next item. + if iter.Key() == page.Marker { + iter.Next() + } } var cnt int64 = 0 @@ -102,18 +105,18 @@ func (db *Backend) ListBucket(name string, prefix *gofakes3.Prefix, page gofakes for iter.Next() { item := iter.Value().(*bucketObject) - if !prefix.Match(item.data.name, &match) { + switch { + case !prefix.Match(item.data.name, &match): continue - } else if item.data.deleteMarker { + case item.data.deleteMarker: continue - } else if match.CommonPrefix { + case match.CommonPrefix: if match.MatchedPart == lastMatchedPart { continue // Should not count towards keys } response.AddPrefix(match.MatchedPart) lastMatchedPart = match.MatchedPart - - } else { + default: response.Add(&gofakes3.Content{ Key: item.data.name, LastModified: gofakes3.NewContentTime(item.data.lastModified), @@ -261,6 +264,10 @@ func (db *Backend) PutObject(bucketName, objectName string, meta map[string]stri return result, nil } +func (db *Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + return gofakes3.CopyObject(db, srcBucket, srcKey, dstBucket, dstKey, meta) +} + func (db *Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { db.lock.Lock() defer db.lock.Unlock() diff --git a/backend/s3mem/bucket.go b/backend/s3mem/bucket.go index 1ffea7ef..b3497a3c 100644 --- a/backend/s3mem/bucket.go +++ b/backend/s3mem/bucket.go @@ -143,7 +143,7 @@ func (bi *bucketData) toObject(rangeRequest *gofakes3.ObjectRangeRequest, withBo // The data slice should be completely replaced if the bucket item is edited, so // it should be safe to return the data slice directly. - contents = s3io.ReaderWithDummyCloser{bytes.NewReader(data)} + contents = s3io.ReaderWithDummyCloser{Reader: bytes.NewReader(data)} } else { contents = s3io.NoOpReadCloser{} diff --git a/cmd/gofakes3/main.go b/cmd/gofakes3/main.go index 87b33e67..c29c97eb 100644 --- a/cmd/gofakes3/main.go +++ b/cmd/gofakes3/main.go @@ -10,6 +10,7 @@ import ( httppprof "net/http/pprof" "os" "runtime/pprof" + "strings" "time" "github.com/johannesboyne/gofakes3" @@ -29,21 +30,25 @@ func main() { } type fakeS3Flags struct { - host string - backendKind string - initialBucket string - fixedTimeStr string - noIntegrity bool - hostBucket bool - autoBucket bool - quiet bool - - boltDb string - directFsPath string - directFsMeta string - directFsBucket string - fsPath string - fsMeta string + host string + backendKind string + initialBucket string + fixedTimeStr string + noIntegrity bool + hostBucket bool + hostBucketBases HostList + autoBucket bool + quiet bool + + boltDb string + directFsPath string + directFsMeta string + directFsBucket string + directFsCreatePaths bool + + fsPath string + fsMeta string + fsCreatePaths bool debugCPU string debugHost string @@ -54,8 +59,17 @@ func (f *fakeS3Flags) attach(flagSet *flag.FlagSet) { flagSet.StringVar(&f.fixedTimeStr, "time", "", "RFC3339 format. If passed, the server's clock will always see this time; does not affect existing stored dates.") flagSet.StringVar(&f.initialBucket, "initialbucket", "", "If passed, this bucket will be created on startup if it does not already exist.") flagSet.BoolVar(&f.noIntegrity, "no-integrity", false, "Pass this flag to disable Content-MD5 validation when uploading.") - flagSet.BoolVar(&f.hostBucket, "hostbucket", false, "If passed, the bucket name will be extracted from the first segment of the hostname, rather than the first part of the URL path.") flagSet.BoolVar(&f.autoBucket, "autobucket", false, "If passed, nonexistent buckets will be created on first use instead of raising an error") + flagSet.BoolVar(&f.hostBucket, "hostbucket", false, ""+ + "If passed, the bucket name will be extracted from the first segment of the hostname, "+ + "rather than the first part of the URL path. Disables path-based mode. If you require both, use "+ + "-hostbucketbase.") + flagSet.Var(&f.hostBucketBases, "hostbucketbase", ""+ + "If passed, the bucket name will be presumed to be the hostname segment before the "+ + "host bucket base, i.e. if hostbucketbase is 'example.com' and you request 'foo.example.com', "+ + "the bucket is presumed to be 'foo'. Any other hostname not matching this pattern will use "+ + "path routing. Takes precedence over -hostbucket. Can be passed multiple times, or as a single "+ + "comma separated list") // Logging flagSet.BoolVar(&f.quiet, "quiet", false, "If passed, log messages are not printed to stderr") @@ -63,11 +77,15 @@ func (f *fakeS3Flags) attach(flagSet *flag.FlagSet) { // Backend specific: flagSet.StringVar(&f.backendKind, "backend", "", "Backend to use to store data (memory, bolt, directfs, fs)") flagSet.StringVar(&f.boltDb, "bolt.db", "locals3.db", "Database path / name when using bolt backend") + flagSet.StringVar(&f.directFsPath, "directfs.path", "", "File path to serve using S3. You should not modify the contents of this path outside gofakes3 while it is running as it can cause inconsistencies.") flagSet.StringVar(&f.directFsMeta, "directfs.meta", "", "Optional path for storing S3 metadata for your bucket. If not passed, metadata will not persist between restarts of gofakes3.") flagSet.StringVar(&f.directFsBucket, "directfs.bucket", "mybucket", "Name of the bucket for your file path; this will be the only supported bucket by the 'directfs' backend for the duration of your run.") + flagSet.BoolVar(&f.directFsCreatePaths, "directfs.create", false, "Create all paths for direct filesystem backend") + flagSet.StringVar(&f.fsPath, "fs.path", "", "Path to your S3 buckets. Buckets are stored under the '/buckets' subpath.") flagSet.StringVar(&f.fsMeta, "fs.meta", "", "Optional path for storing S3 metadata for your buckets. Defaults to the '/metadata' subfolder of -fs.path if not passed.") + flagSet.BoolVar(&f.fsCreatePaths, "fs.create", false, "Create all paths for filesystem backends") // Debugging: flagSet.StringVar(&f.debugHost, "debug.host", "", "Run the debug server on this host") @@ -78,6 +96,20 @@ func (f *fakeS3Flags) attach(flagSet *flag.FlagSet) { flagSet.StringVar(&f.initialBucket, "bucket", "", `Deprecated; use -initialbucket`) } +func (f *fakeS3Flags) fsPathFlags() (flags s3afero.FsFlags) { + if f.fsCreatePaths { + flags |= s3afero.FsPathCreateAll + } + return flags +} + +func (f *fakeS3Flags) directFsPathFlags() (flags s3afero.FsFlags) { + if f.directFsCreatePaths { + flags |= s3afero.FsPathCreateAll + } + return flags +} + func (f *fakeS3Flags) timeOptions() (source gofakes3.TimeSource, skewLimit time.Duration, err error) { skewLimit = gofakes3.DefaultSkewLimit @@ -163,14 +195,14 @@ func run() error { log.Println("warning: time source not supported by this backend") } - baseFs, err := s3afero.FsPath(values.fsPath) + baseFs, err := s3afero.FsPath(values.fsPath, values.fsPathFlags()) if err != nil { return fmt.Errorf("gofakes3: could not create -fs.path: %v", err) } var options []s3afero.MultiOption if values.fsMeta != "" { - metaFs, err := s3afero.FsPath(values.fsMeta) + metaFs, err := s3afero.FsPath(values.fsMeta, values.fsPathFlags()) if err != nil { return fmt.Errorf("gofakes3: could not create -fs.meta: %v", err) } @@ -193,14 +225,14 @@ func run() error { log.Println("warning: time source not supported by this backend") } - baseFs, err := s3afero.FsPath(values.directFsPath) + baseFs, err := s3afero.FsPath(values.directFsPath, values.directFsPathFlags()) if err != nil { return fmt.Errorf("gofakes3: could not create -directfs.path: %v", err) } var metaFs afero.Fs if values.directFsMeta != "" { - metaFs, err = s3afero.FsPath(values.directFsMeta) + metaFs, err = s3afero.FsPath(values.directFsMeta, values.directFsPathFlags()) if err != nil { return fmt.Errorf("gofakes3: could not create -directfs.meta: %v", err) } @@ -235,6 +267,7 @@ func run() error { gofakes3.WithTimeSource(timeSource), gofakes3.WithLogger(logger), gofakes3.WithHostBucket(values.hostBucket), + gofakes3.WithHostBucketBase(values.hostBucketBases.Values...), gofakes3.WithAutoBucket(values.autoBucket), ) @@ -269,3 +302,24 @@ func profile(values fakeS3Flags) (func(), error) { return fn, nil } + +type HostList struct { + Values []string +} + +func (sl HostList) String() string { + return strings.Join(sl.Values, ",") +} + +func (sl HostList) Type() string { return "[]string" } + +func (sl *HostList) Set(s string) error { + for _, part := range strings.Split(s, ",") { + part = strings.Trim(strings.TrimSpace(part), ".") + if part == "" { + return fmt.Errorf("host is empty") + } + sl.Values = append(sl.Values, part) + } + return nil +} diff --git a/cors.go b/cors.go index 105c30d8..15c7ff47 100644 --- a/cors.go +++ b/cors.go @@ -11,6 +11,7 @@ var ( "Accept-Encoding", "Authorization", "Content-Disposition", + "Content-Encoding", "Content-Length", "Content-Type", "X-Amz-Date", diff --git a/go.mod b/go.mod index 376d8f73..0e7cb2b5 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,12 @@ module github.com/johannesboyne/gofakes3 go 1.16 require ( - github.com/aws/aws-sdk-go v1.17.4 + github.com/aws/aws-sdk-go v1.44.256 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 - github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 + github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 github.com/spf13/afero v1.2.1 - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.5.1 go.etcd.io/bbolt v1.3.5 - golang.org/x/net v0.0.0-20190310074541-c10a0554eabf // indirect - golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f + golang.org/x/tools v0.8.0 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce - gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index c7dfe9c5..d659e2c5 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,75 @@ -github.com/aws/aws-sdk-go v1.17.4 h1:L2KFocQhg48kIzEAV98SnSz3nmIZ3UDFP+vU647KO3c= -github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= -github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190310074541-c10a0554eabf h1:J7RqX9u0J9ZB37CGaFc2VC+QZZT6E6jnDbrboEFVo0U= -golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f h1:SUQ6L9W8e5xt2GFO9s+i18JGITAfem+a0AQuFU8Ls74= -golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/gofakes3.go b/gofakes3.go index 74864074..0c3d3b05 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -1,7 +1,6 @@ package gofakes3 import ( - "bytes" "encoding/base64" "encoding/hex" "encoding/xml" @@ -28,14 +27,15 @@ type GoFakeS3 struct { storage Backend versioned VersionedBackend - timeSource TimeSource - timeSkew time.Duration - metadataSizeLimit int - integrityCheck bool - failOnUnimplementedPage bool - hostBucket bool - autoBucket bool - uploader *uploader + timeSource TimeSource // WithTimeSource + timeSkew time.Duration // WithTimeSkewLimit + metadataSizeLimit int // WithMetadataSizeLimit + integrityCheck bool // WithIntegrityCheck + failOnUnimplementedPage bool // WithUnimplementedPageError + hostBucket bool // WithHostBucket + hostBucketBases []string // WithHostBucketBase + autoBucket bool // WithAutoBucket + uploader MultipartBackend log Logger } @@ -48,7 +48,6 @@ func New(backend Backend, options ...Option) *GoFakeS3 { timeSkew: DefaultSkewLimit, metadataSizeLimit: DefaultMetadataSizeLimit, integrityCheck: true, - uploader: newUploader(), requestID: 0, } @@ -64,6 +63,11 @@ func New(backend Backend, options ...Option) *GoFakeS3 { if s3.timeSource == nil { s3.timeSource = DefaultTimeSource() } + if mpb, ok := backend.(MultipartBackend); ok { + s3.uploader = mpb + } else { + s3.uploader = newUploader(backend, s3.timeSource) + } return s3 } @@ -80,7 +84,9 @@ func (g *GoFakeS3) Server() http.Handler { handler = g.timeSkewMiddleware(handler) } - if g.hostBucket { + if len(g.hostBucketBases) > 0 { + handler = g.hostBucketBaseMiddleware(handler) + } else if g.hostBucket { handler = g.hostBucketMiddleware(handler) } @@ -124,6 +130,45 @@ func (g *GoFakeS3) hostBucketMiddleware(handler http.Handler) http.Handler { }) } +// hostBucketBaseMiddleware forces the server to use VirtualHost-style bucket URLs: +// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +func (g *GoFakeS3) hostBucketBaseMiddleware(handler http.Handler) http.Handler { + bases := make([]string, len(g.hostBucketBases)) + for idx, base := range g.hostBucketBases { + bases[idx] = "." + strings.Trim(base, ".") + } + + matchBucket := func(host string) (bucket string, ok bool) { + for _, base := range bases { + if !strings.HasSuffix(host, base) { + continue + } + bucket = host[:len(host)-len(base)] + if idx := strings.IndexByte(bucket, '.'); idx >= 0 { + continue + } + return bucket, true + } + return "", false + } + + return http.HandlerFunc(func(w http.ResponseWriter, rq *http.Request) { + bucket, ok := matchBucket(rq.Host) + if !ok { + handler.ServeHTTP(w, rq) + return + } + p := rq.URL.Path + rq.URL.Path = "/" + bucket + if p != "/" { + rq.URL.Path += p + } + g.log.Print(LogInfo, p, "=>", rq.URL) + + handler.ServeHTTP(w, rq) + }) +} + func (g *GoFakeS3) httpError(w http.ResponseWriter, r *http.Request, err error) { resp := ensureErrorResponse(err, "") // FIXME: request id if resp.ErrorCode() == ErrInternal { @@ -461,6 +506,12 @@ func (g *GoFakeS3) writeGetOrHeadObjectResponse(obj *Object, w http.ResponseWrit return ErrNotModified } + lastModified, _ := time.Parse(http.TimeFormat, obj.Metadata["Last-Modified"]) + ifModifiedSince, _ := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")) + if !lastModified.IsZero() && !ifModifiedSince.Before(lastModified) { + return ErrNotModified + } + w.Header().Set("Accept-Ranges", "bytes") return nil @@ -659,17 +710,11 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h if err != nil { return err } - srcObj, err := g.storage.GetObject(srcBucket, srcKey, nil) + srcObj, err := g.storage.HeadObject(srcBucket, srcKey) if err != nil { return err } - if srcObj == nil { - g.log.Print(LogErr, "unexpected nil object for key", bucket, object) - return ErrInternal - } - defer srcObj.Contents.Close() - // XXX No support for delete marker // "If the current version of the object is a delete marker, Amazon S3 // behaves as if the object was deleted." @@ -689,15 +734,11 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h if srcObj.VersionID != "" { w.Header().Set("x-amz-copy-source-version-id", string(srcObj.VersionID)) } - if result.VersionID != "" { - g.log.Print(LogInfo, "CREATED VERSION:", bucket, object, result.VersionID) - w.Header().Set("x-amz-version-id", string(result.VersionID)) + if srcObj.VersionID != "" { + w.Header().Set("x-amz-version-id", string(srcObj.VersionID)) } - return g.xmlEncoder(w).Encode(CopyObjectResult{ - ETag: `"` + hex.EncodeToString(srcObj.Hash) + `"`, - LastModified: NewContentTime(g.timeSource.Now()), - }) + return g.xmlEncoder(w).Encode(result) } func (g *GoFakeS3) deleteObject(bucket, object string, w http.ResponseWriter, r *http.Request) error { @@ -807,9 +848,12 @@ func (g *GoFakeS3) initiateMultipartUpload(bucket, object string, w http.Respons return err } - upload := g.uploader.Begin(bucket, object, meta, g.timeSource.Now()) + uploadID, err := g.uploader.CreateMultipartUpload(bucket, object, meta) + if err != nil { + return err + } out := InitiateMultipartUpload{ - UploadID: upload.ID, + UploadID: uploadID, Bucket: bucket, Key: object, } @@ -836,15 +880,6 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload return ErrMissingContentLength } - upload, err := g.uploader.Get(bucket, object, uploadID) - if err != nil { - // FIXME: What happens with S3 when you abort a multipart upload while - // part uploads are still in progress? In this case, we will retain the - // reference to the part even though another request goroutine may - // delete it; it will be available for GC when this function finishes. - return err - } - defer r.Body.Close() var rdr io.Reader = r.Body @@ -863,16 +898,7 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload } } - body, err := ReadAll(rdr, size) - if err != nil { - return err - } - - if int64(len(body)) != r.ContentLength { - return ErrIncompleteBody - } - - etag, err := upload.AddPart(int(partNumber), g.timeSource.Now(), body) + etag, err := g.uploader.UploadPart(bucket, object, uploadID, int(partNumber), r.ContentLength, rdr) if err != nil { return err } @@ -944,7 +970,7 @@ func (g *GoFakeS3) getObjectTags(bucket, object string, version string, w http.R func (g *GoFakeS3) abortMultipartUpload(bucket, object string, uploadID UploadID, w http.ResponseWriter, r *http.Request) error { g.log.Print(LogInfo, "abort multipart upload", bucket, object, uploadID) - if _, err := g.uploader.Complete(bucket, object, uploadID); err != nil { + if err := g.uploader.AbortMultipartUpload(bucket, object, uploadID); err != nil { return err } w.WriteHeader(http.StatusNoContent) @@ -959,12 +985,7 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - upload, err := g.uploader.Complete(bucket, object, uploadID) - if err != nil { - return err - } - - fileBody, etag, err := upload.Reassemble(&in) + versionID, etag, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in) if err != nil { return err } @@ -1001,7 +1022,7 @@ func (g *GoFakeS3) listMultipartUploads(bucket string, w http.ResponseWriter, r maxUploads = DefaultMaxUploads } - out, err := g.uploader.List(bucket, marker, prefix, maxUploads) + out, err := g.uploader.ListMultipartUploads(bucket, marker, prefix, maxUploads) if err != nil { return err } @@ -1137,7 +1158,10 @@ func metadataSize(meta map[string]string) int { func metadataHeaders(headers map[string][]string, at time.Time, sizeLimit int) (map[string]string, error) { meta := make(map[string]string) for hk, hv := range headers { - if strings.HasPrefix(hk, "X-Amz-") || hk == "Content-Type" || hk == "Content-Disposition" { + if strings.HasPrefix(hk, "X-Amz-") || + hk == "Content-Type" || + hk == "Content-Disposition" || + hk == "Content-Encoding" { meta[hk] = hv[0] } } diff --git a/gofakes3_internal_test.go b/gofakes3_internal_test.go index 8759d12a..609597b1 100644 --- a/gofakes3_internal_test.go +++ b/gofakes3_internal_test.go @@ -82,6 +82,39 @@ func TestHostBucketMiddleware(t *testing.T) { } } +func TestHostBucketBaseMiddleware(t *testing.T) { + for _, tc := range []struct { + bases []string + host string + inPath string + outPath string + }{ + {[]string{"localhost"}, "foo", "/", "/"}, + {[]string{"localhost"}, "localhost", "/", "/"}, + {[]string{"localhost"}, "mybucket.fleebderb", "/", "/"}, + {[]string{"localhost"}, "mybucket.localhost", "/", "/mybucket"}, + {[]string{"localhost"}, "mybucket.localhost", "/object", "/mybucket/object"}, + } { + t.Run("", func(t *testing.T) { + var g GoFakeS3 + g.hostBucketBases = tc.bases + g.log = DiscardLog() + + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != tc.outPath { + t.Fatal(r.URL.Path, "!=", tc.outPath) + } + }) + + handler := g.hostBucketBaseMiddleware(inner) + rq := httptest.NewRequest("GET", tc.inPath, nil) + rq.Host = tc.host + rs := httptest.NewRecorder() + handler.ServeHTTP(rs, rq) + }) + } +} + type failingResponseWriter struct { *httptest.ResponseRecorder } diff --git a/gofakes3_test.go b/gofakes3_test.go index e69481c7..abb340fd 100644 --- a/gofakes3_test.go +++ b/gofakes3_test.go @@ -277,6 +277,37 @@ func TestCreateObjectWithContentDisposition(t *testing.T) { } } +func TestCreateObjectWithContentEncoding(t *testing.T) { + ts := newTestServer(t) + defer ts.Close() + svc := ts.s3Client() + + _, err := svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(defaultBucket), + Key: aws.String("object"), + Body: bytes.NewReader([]byte{ // "hello", gzipped + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xcb, 0x48, + 0xcd, 0xc9, 0xc9, 0x07, 0x00, 0x86, 0xa6, 0x10, 0x36, 0x05, 0x00, 0x00, + 0x00, + }), + ContentType: aws.String("text/plain"), + ContentEncoding: aws.String("gzip"), + }) + ts.OK(err) + + obj, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(defaultBucket), + Key: aws.String("object"), + }) + content, err := io.ReadAll(obj.Body) + if err != nil { + t.Fatal("error reading body", err) + } + if !bytes.Equal(content, []byte("hello")) { + t.Fatal("incorrect body with Content-Encoding: gzip") + } +} + func TestCreateObjectMetadataAndObjectTagging(t *testing.T) { ts := newTestServer(t) defer ts.Close() diff --git a/option.go b/option.go index ed3a9222..84fc484c 100644 --- a/option.go +++ b/option.go @@ -19,7 +19,6 @@ func WithTimeSource(timeSource TimeSource) Option { // calculate the skew. // // See DefaultSkewLimit for the starting value, set to '0' to disable. -// func WithTimeSkewLimit(skew time.Duration) Option { return func(g *GoFakeS3) { g.timeSkew = skew } } @@ -60,12 +59,28 @@ func WithRequestID(id uint64) Option { // If active, the URL 'http://mybucket.localhost/object' will be routed // as if the URL path was '/mybucket/object'. // -// See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html -// for details. +// This will apply to all requests. If you want to be more specific, provide +// WithHostBucketBase. +// +// See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html for details. func WithHostBucket(enabled bool) Option { return func(g *GoFakeS3) { g.hostBucket = enabled } } +// WithHostBucketBase enables or disables bucket rewriting in the router, but only if the +// host is a subdomain of the base. +// +// If set to 'example.com', the URL 'http://mybucket.example.com/object' will be routed as +// if the URL path was '/mybucket/object', but 'http://example.com/bucket/object' will use +// path-based bucket routing instead. +// +// You may pass multiple bases, they are tested in order. +// +// See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html for details. +func WithHostBucketBase(hosts ...string) Option { + return func(g *GoFakeS3) { g.hostBucketBases = hosts } +} + // WithoutVersioning disables versioning on the passed backend, if it supported it. func WithoutVersioning() Option { return func(g *GoFakeS3) { g.versioned = nil } @@ -84,5 +99,5 @@ func WithUnimplementedPageError() Option { // WithAutoBucket instructs GoFakeS3 to create buckets that don't exist on first use, // rather than returning ErrNoSuchBucket. func WithAutoBucket(enabled bool) Option { - return func(g *GoFakeS3) { g.autoBucket = true } + return func(g *GoFakeS3) { g.autoBucket = enabled } } diff --git a/prefix.go b/prefix.go index 33082fde..67c47c40 100644 --- a/prefix.go +++ b/prefix.go @@ -51,7 +51,7 @@ func NewFolderPrefix(prefix string) (p Prefix) { // func (p Prefix) FilePrefix() (path, remaining string, ok bool) { if !p.HasPrefix || !p.HasDelimiter || p.Delimiter != "/" { - return "", "", ok + return "", "", p.Delimiter == "/" } idx := strings.LastIndexByte(p.Prefix, '/') diff --git a/uploader.go b/uploader.go index 3ba99626..f5f92ec6 100644 --- a/uploader.go +++ b/uploader.go @@ -1,9 +1,11 @@ package gofakes3 import ( + "bytes" "crypto/md5" "encoding/hex" "fmt" + "io" "math/big" "net/url" "strings" @@ -14,6 +16,8 @@ import ( "github.com/ryszard/goskiplist/skiplist" ) +var _ MultipartBackend = &uploader{} + var add1 = new(big.Int).SetInt64(1) /* @@ -146,8 +150,9 @@ func (bu *bucketUploads) remove(uploadID UploadID) { // good convenience for Backend implementers if their use case did not require // persistent multipart upload handling, or it could be satisfied by this // naive implementation. -// type uploader struct { + timeSource TimeSource + storage Backend // uploadIDs use a big.Int to allow unbounded IDs (not that you'd be // expected to ever generate 4.2 billion of these but who are we to judge?) uploadID *big.Int @@ -156,14 +161,16 @@ type uploader struct { mu sync.Mutex } -func newUploader() *uploader { +func newUploader(b Backend, timeSource TimeSource) *uploader { return &uploader{ - buckets: make(map[string]*bucketUploads), - uploadID: new(big.Int), + buckets: make(map[string]*bucketUploads), + storage: b, + timeSource: timeSource, + uploadID: new(big.Int), } } -func (u *uploader) Begin(bucket, object string, meta map[string]string, initiated time.Time) *multipartUpload { +func (u *uploader) CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error) { u.mu.Lock() defer u.mu.Unlock() @@ -174,7 +181,7 @@ func (u *uploader) Begin(bucket, object string, meta map[string]string, initiate Bucket: bucket, Object: object, Meta: meta, - Initiated: initiated, + Initiated: u.timeSource.Now(), } // FIXME: make sure the uploader responds to DeleteBucket @@ -186,7 +193,7 @@ func (u *uploader) Begin(bucket, object string, meta map[string]string, initiate bucketUploads.add(mpu) - return mpu + return mpu.ID, nil } func (u *uploader) ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) { @@ -232,7 +239,7 @@ func (u *uploader) ListParts(bucket, object string, uploadID UploadID, marker in return &result, nil } -func (u *uploader) List(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) { +func (u *uploader) ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) { u.mu.Lock() defer u.mu.Unlock() @@ -345,24 +352,110 @@ done: return &result, nil } -func (u *uploader) Complete(bucket, object string, id UploadID) (*multipartUpload, error) { +func (u *uploader) AbortMultipartUpload(bucket, object string, id UploadID) error { u.mu.Lock() defer u.mu.Unlock() - up, err := u.getUnlocked(bucket, object, id) + _, err := u.getUnlocked(bucket, object, id) if err != nil { - return nil, err + return err } // if getUnlocked succeeded, so will this: u.buckets[bucket].remove(id) - return up, nil + return nil } -func (u *uploader) Get(bucket, object string, id UploadID) (mu *multipartUpload, err error) { - u.mu.Lock() - defer u.mu.Unlock() - return u.getUnlocked(bucket, object, id) +func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) { + if partNumber > MaxUploadPartNumber { + return "", ErrInvalidPart + } + body, err := io.ReadAll(input) + if err != nil { + return "", err + } + if len(body) != int(contentLength) { + return "", ErrIncompleteBody + } + mpu, err := u.getUnlocked(bucket, object, id) + if err != nil { + return "", err + } + + mpu.mu.Lock() + defer mpu.mu.Unlock() + + // What the ETag actually is is not specified, so let's just invent any old thing + // from guaranteed unique input: + hash := md5.New() + hash.Write([]byte(body)) + etag = fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) + + part := multipartUploadPart{ + PartNumber: partNumber, + Body: body, + ETag: etag, + LastModified: NewContentTime(u.timeSource.Now()), + } + if partNumber >= len(mpu.parts) { + mpu.parts = append(mpu.parts, make([]*multipartUploadPart, partNumber-len(mpu.parts)+1)...) + } + mpu.parts[partNumber] = &part + return etag, nil +} + +func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (version VersionID, etag string, err error) { + mpu, err := u.getUnlocked(bucket, object, id) + if err != nil { + return "", "", err + } + + mpu.mu.Lock() + defer mpu.mu.Unlock() + + mpuPartsLen := len(mpu.parts) + + // FIXME: what does AWS do when mpu.Parts > input.Parts? Presumably you may + // end up uploading more parts than you need to assemble, so it should + // probably just ignore that? + if len(input.Parts) > mpuPartsLen { + return "", "", ErrInvalidPart + } + + if !input.partsAreSorted() { + return "", "", ErrInvalidPartOrder + } + + var size int64 + + for _, inPart := range input.Parts { + if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil { + return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) + } + + upPart := mpu.parts[inPart.PartNumber] + if strings.Trim(inPart.ETag, "\"") != strings.Trim(upPart.ETag, "\"") { + return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) + } + + size += int64(len(upPart.Body)) + } + + body := make([]byte, 0, size) + for _, part := range input.Parts { + body = append(body, mpu.parts[part.PartNumber].Body...) + } + + hash := fmt.Sprintf("%x", md5.Sum(body)) + + result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body))) + if err != nil { + return "", "", err + } + + // if getUnlocked succeeded, so will this: + u.buckets[bucket].remove(id) + return result.VersionID, hash, nil } func (u *uploader) getUnlocked(bucket, object string, id UploadID) (mu *multipartUpload, err error) { @@ -447,72 +540,3 @@ type multipartUpload struct { mu sync.Mutex } - -func (mpu *multipartUpload) AddPart(partNumber int, at time.Time, body []byte) (etag string, err error) { - if partNumber > MaxUploadPartNumber { - return "", ErrInvalidPart - } - - mpu.mu.Lock() - defer mpu.mu.Unlock() - - // What the ETag actually is is not specified, so let's just invent any old thing - // from guaranteed unique input: - hash := md5.New() - hash.Write([]byte(body)) - etag = fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) - - part := multipartUploadPart{ - PartNumber: partNumber, - Body: body, - ETag: etag, - LastModified: NewContentTime(at), - } - if partNumber >= len(mpu.parts) { - mpu.parts = append(mpu.parts, make([]*multipartUploadPart, partNumber-len(mpu.parts)+1)...) - } - mpu.parts[partNumber] = &part - return etag, nil -} - -func (mpu *multipartUpload) Reassemble(input *CompleteMultipartUploadRequest) (body []byte, etag string, err error) { - mpu.mu.Lock() - defer mpu.mu.Unlock() - - mpuPartsLen := len(mpu.parts) - - // FIXME: what does AWS do when mpu.Parts > input.Parts? Presumably you may - // end up uploading more parts than you need to assemble, so it should - // probably just ignore that? - if len(input.Parts) > mpuPartsLen { - return nil, "", ErrInvalidPart - } - - if !input.partsAreSorted() { - return nil, "", ErrInvalidPartOrder - } - - var size int64 - - for _, inPart := range input.Parts { - if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil { - return nil, "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) - } - - upPart := mpu.parts[inPart.PartNumber] - if strings.Trim(inPart.ETag, "\"") != strings.Trim(upPart.ETag, "\"") { - return nil, "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) - } - - size += int64(len(upPart.Body)) - } - - body = make([]byte, 0, size) - for _, part := range input.Parts { - body = append(body, mpu.parts[part.PartNumber].Body...) - } - - hash := fmt.Sprintf("%x", md5.Sum(body)) - - return body, hash, nil -}