From b6410d91287136c1185b378c68a826c977ff09f8 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 20 Dec 2024 12:14:21 +0800 Subject: [PATCH] :arrow_up: Upgrade to AWS SDK for Go v2 https://github.com/siyuan-note/siyuan/issues/13557 --- cloud/s3.go | 101 +++++++++++++++++++++++++++++++--------------------- go.mod | 21 +++++++++-- go.sum | 45 ++++++++++++++++++----- 3 files changed, 115 insertions(+), 52 deletions(-) diff --git a/cloud/s3.go b/cloud/s3.go index 93eadc1..5bf45fd 100644 --- a/cloud/s3.go +++ b/cloud/s3.go @@ -19,6 +19,7 @@ package cloud import ( "bytes" "context" + "errors" "io" "math" "net/http" @@ -31,11 +32,12 @@ import ( "time" "github.com/88250/gulu" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - as3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + as3 "github.com/aws/aws-sdk-go-v2/service/s3" + as3Types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/panjf2000/ants/v2" "github.com/siyuan-note/dejavu/entity" "github.com/siyuan-note/logging" @@ -82,7 +84,7 @@ func (s3 *S3) UploadObject(filePath string, overwrite bool) (length int64, err e } defer file.Close() key := path.Join("repo", filePath) - _, err = svc.PutObjectWithContext(ctx, &as3.PutObjectInput{ + _, err = svc.PutObject(ctx, &as3.PutObjectInput{ Bucket: aws.String(s3.Conf.S3.Bucket), Key: aws.String(key), CacheControl: aws.String("no-cache"), @@ -103,7 +105,7 @@ func (s3 *S3) UploadBytes(filePath string, data []byte, overwrite bool) (length defer cancelFn() key := path.Join("repo", filePath) - _, err = svc.PutObjectWithContext(ctx, &as3.PutObjectInput{ + _, err = svc.PutObject(ctx, &as3.PutObjectInput{ Bucket: aws.String(s3.Conf.S3.Bucket), Key: aws.String(key), CacheControl: aws.String("no-cache"), @@ -127,7 +129,7 @@ func (s3 *S3) DownloadObject(filePath string) (data []byte, err error) { Key: aws.String(key), ResponseCacheControl: aws.String("no-cache"), } - resp, err := svc.GetObjectWithContext(ctx, input) + resp, err := svc.GetObject(ctx, input) if nil != err { if s3.isErrNotFound(err) { err = ErrCloudObjectNotFound @@ -149,7 +151,7 @@ func (s3 *S3) RemoveObject(key string) (err error) { svc := s3.getService() ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) defer cancelFn() - _, err = svc.DeleteObjectWithContext(ctx, &as3.DeleteObjectInput{ + _, err = svc.DeleteObject(ctx, &as3.DeleteObjectInput{ Bucket: aws.String(s3.Conf.S3.Bucket), Key: aws.String(key), }) @@ -304,15 +306,23 @@ func (s3 *S3) ListObjects(pathPrefix string) (ret map[string]*entity.ObjectInfo, if endWithSlash { pathPrefix += "/" } - limit := int64(1000) + limit := int32(1000) ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) defer cancelFn() - err = svc.ListObjectsPagesWithContext(ctx, &as3.ListObjectsInput{ + paginator := as3.NewListObjectsV2Paginator(svc, &as3.ListObjectsV2Input{ Bucket: &s3.Conf.S3.Bucket, Prefix: &pathPrefix, MaxKeys: &limit, - }, func(output *as3.ListObjectsOutput, last bool) bool { + }) + + for paginator.HasMorePages() { + output, pErr := paginator.NextPage(ctx) + if nil != pErr { + logging.LogErrorf("list objects failed: %s", pErr) + return nil, pErr + } + for _, entry := range output.Contents { filePath := strings.TrimPrefix(*entry.Key, pathPrefix) ret[filePath] = &entity.ObjectInfo{ @@ -320,12 +330,6 @@ func (s3 *S3) ListObjects(pathPrefix string) (ret map[string]*entity.ObjectInfo, Size: *entry.Size, } } - return true - }) - - if nil != err { - logging.LogErrorf("list objects failed: %s", err) - return } return } @@ -358,11 +362,14 @@ func (s3 *S3) repoIndex(id string) (ret *entity.Index, err error) { func (s3 *S3) listRepoRefs(refPrefix string) (ret []*Ref, err error) { svc := s3.getService() + ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) + defer cancelFn() + prefix := path.Join("repo", "refs", refPrefix) - limit := int64(32) + limit := int32(32) marker := "" for { - output, listErr := svc.ListObjects(&as3.ListObjectsInput{ + output, listErr := svc.ListObjects(ctx, &as3.ListObjectsInput{ Bucket: &s3.Conf.S3.Bucket, Prefix: &prefix, Marker: &marker, @@ -408,7 +415,10 @@ func (s3 *S3) listRepoRefs(refPrefix string) (ret []*Ref, err error) { func (s3 *S3) listRepos() (ret []*Repo, err error) { svc := s3.getService() - output, err := svc.ListBuckets(&as3.ListBucketsInput{}) + ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) + defer cancelFn() + + output, err := svc.ListBuckets(ctx, &as3.ListBucketsInput{}) if nil != err { return } @@ -427,8 +437,10 @@ func (s3 *S3) listRepos() (ret []*Repo, err error) { func (s3 *S3) statFile(key string) (info *objectInfo, err error) { svc := s3.getService() + ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) + defer cancelFn() - header, err := svc.HeadObject(&as3.HeadObjectInput{ + header, err := svc.HeadObject(ctx, &as3.HeadObjectInput{ Bucket: &s3.Conf.S3.Bucket, Key: &key, }) @@ -479,27 +491,36 @@ func (s3 *S3) getNotFound(keys []string) (ret []string, err error) { return } -func (s3 *S3) getService() *as3.S3 { - sess := session.Must(session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(s3.Conf.S3.AccessKey, s3.Conf.S3.SecretKey, ""), - Endpoint: aws.String(s3.Conf.S3.Endpoint), - Region: aws.String(s3.Conf.S3.Region), - S3ForcePathStyle: aws.Bool(s3.Conf.S3.PathStyle), - HTTPClient: s3.HTTPClient, - })) - return as3.New(sess) +func (s3 *S3) getService() *as3.Client { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + logging.LogErrorf("load default config failed: %s", err) + } + + return as3.NewFromConfig(cfg, func(o *as3.Options) { + o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(s3.Conf.S3.AccessKey, s3.Conf.S3.SecretKey, "")) + o.BaseEndpoint = aws.String(s3.Conf.S3.Endpoint) + o.Region = s3.Conf.S3.Region + o.UsePathStyle = s3.Conf.S3.PathStyle + o.HTTPClient = s3.HTTPClient + }) } func (s3 *S3) isErrNotFound(err error) bool { - switch err.(type) { - case awserr.Error: - code := err.(awserr.Error).Code() - switch code { - case as3.ErrCodeNoSuchKey: - return true - } + var nsk *as3Types.NoSuchKey + if errors.As(err, &nsk) { + return true + } + + var nf *as3Types.NotFound + if errors.As(err, &nf) { + return true } - msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "does not exist") || strings.Contains(msg, "404") || strings.Contains(msg, "no such file or directory") + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + msg := strings.ToLower(apiErr.ErrorMessage()) + return strings.Contains(msg, "does not exist") || strings.Contains(msg, "404") || strings.Contains(msg, "no such file or directory") + } + return false } diff --git a/go.mod b/go.mod index d2265d6..3b39f72 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,11 @@ toolchain go1.23.1 require ( github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7 github.com/88250/gulu v1.2.3-0.20241212012748-c4dc08fe45ec - github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go-v2 v1.32.7 + github.com/aws/aws-sdk-go-v2/config v1.28.7 + github.com/aws/aws-sdk-go-v2/credentials v1.17.48 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 + github.com/aws/smithy-go v1.22.1 github.com/dgraph-io/ristretto v1.0.0 github.com/klauspost/compress v1.17.11 github.com/panjf2000/ants/v2 v2.10.0 @@ -25,6 +29,19 @@ require ( require ( github.com/andybalholm/brotli v1.1.1 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -34,7 +51,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imroc/req/v3 v3.49.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/onsi/ginkgo/v2 v2.22.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect @@ -49,7 +65,6 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.28.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) //replace github.com/siyuan-note/filelock => D:\88250\filelock diff --git a/go.sum b/go.sum index 23f6e8a..8f128b6 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,42 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= +github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= +github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= @@ -48,10 +82,6 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/imroc/req/v3 v3.49.0 h1:5Rac2qvz7Dq0E3PeBo/c2szV3hagPQIGLoHtfBmYhu4= github.com/imroc/req/v3 v3.49.0/go.mod h1:XZf4t94DNJzcA0UOBlA68hmSrWsAyvN407ADdH4mzCA= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -186,9 +216,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=