Skip to content

Commit

Permalink
⬆️ Upgrade to AWS SDK for Go v2 siyuan-note/siyuan#13557
Browse files Browse the repository at this point in the history
  • Loading branch information
88250 committed Dec 20, 2024
1 parent a9e14d8 commit b6410d9
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 52 deletions.
101 changes: 61 additions & 40 deletions cloud/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package cloud
import (
"bytes"
"context"
"errors"
"io"
"math"
"net/http"
Expand All @@ -31,11 +32,12 @@ import (
"time"

"github.com/88250/gulu"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
as3 "github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
as3 "github.com/aws/aws-sdk-go-v2/service/s3"
as3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/panjf2000/ants/v2"
"github.com/siyuan-note/dejavu/entity"
"github.com/siyuan-note/logging"
Expand Down Expand Up @@ -82,7 +84,7 @@ func (s3 *S3) UploadObject(filePath string, overwrite bool) (length int64, err e
}
defer file.Close()
key := path.Join("repo", filePath)
_, err = svc.PutObjectWithContext(ctx, &as3.PutObjectInput{
_, err = svc.PutObject(ctx, &as3.PutObjectInput{
Bucket: aws.String(s3.Conf.S3.Bucket),
Key: aws.String(key),
CacheControl: aws.String("no-cache"),
Expand All @@ -103,7 +105,7 @@ func (s3 *S3) UploadBytes(filePath string, data []byte, overwrite bool) (length
defer cancelFn()

key := path.Join("repo", filePath)
_, err = svc.PutObjectWithContext(ctx, &as3.PutObjectInput{
_, err = svc.PutObject(ctx, &as3.PutObjectInput{
Bucket: aws.String(s3.Conf.S3.Bucket),
Key: aws.String(key),
CacheControl: aws.String("no-cache"),
Expand All @@ -127,7 +129,7 @@ func (s3 *S3) DownloadObject(filePath string) (data []byte, err error) {
Key: aws.String(key),
ResponseCacheControl: aws.String("no-cache"),
}
resp, err := svc.GetObjectWithContext(ctx, input)
resp, err := svc.GetObject(ctx, input)
if nil != err {
if s3.isErrNotFound(err) {
err = ErrCloudObjectNotFound
Expand All @@ -149,7 +151,7 @@ func (s3 *S3) RemoveObject(key string) (err error) {
svc := s3.getService()
ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second)
defer cancelFn()
_, err = svc.DeleteObjectWithContext(ctx, &as3.DeleteObjectInput{
_, err = svc.DeleteObject(ctx, &as3.DeleteObjectInput{
Bucket: aws.String(s3.Conf.S3.Bucket),
Key: aws.String(key),
})
Expand Down Expand Up @@ -304,28 +306,30 @@ func (s3 *S3) ListObjects(pathPrefix string) (ret map[string]*entity.ObjectInfo,
if endWithSlash {
pathPrefix += "/"
}
limit := int64(1000)
limit := int32(1000)
ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second)
defer cancelFn()

err = svc.ListObjectsPagesWithContext(ctx, &as3.ListObjectsInput{
paginator := as3.NewListObjectsV2Paginator(svc, &as3.ListObjectsV2Input{
Bucket: &s3.Conf.S3.Bucket,
Prefix: &pathPrefix,
MaxKeys: &limit,
}, func(output *as3.ListObjectsOutput, last bool) bool {
})

for paginator.HasMorePages() {
output, pErr := paginator.NextPage(ctx)
if nil != pErr {
logging.LogErrorf("list objects failed: %s", pErr)
return nil, pErr
}

for _, entry := range output.Contents {
filePath := strings.TrimPrefix(*entry.Key, pathPrefix)
ret[filePath] = &entity.ObjectInfo{
Path: filePath,
Size: *entry.Size,
}
}
return true
})

if nil != err {
logging.LogErrorf("list objects failed: %s", err)
return
}
return
}
Expand Down Expand Up @@ -358,11 +362,14 @@ func (s3 *S3) repoIndex(id string) (ret *entity.Index, err error) {

func (s3 *S3) listRepoRefs(refPrefix string) (ret []*Ref, err error) {
svc := s3.getService()
ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second)
defer cancelFn()

prefix := path.Join("repo", "refs", refPrefix)
limit := int64(32)
limit := int32(32)
marker := ""
for {
output, listErr := svc.ListObjects(&as3.ListObjectsInput{
output, listErr := svc.ListObjects(ctx, &as3.ListObjectsInput{
Bucket: &s3.Conf.S3.Bucket,
Prefix: &prefix,
Marker: &marker,
Expand Down Expand Up @@ -408,7 +415,10 @@ func (s3 *S3) listRepoRefs(refPrefix string) (ret []*Ref, err error) {

func (s3 *S3) listRepos() (ret []*Repo, err error) {
svc := s3.getService()
output, err := svc.ListBuckets(&as3.ListBucketsInput{})
ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second)
defer cancelFn()

output, err := svc.ListBuckets(ctx, &as3.ListBucketsInput{})
if nil != err {
return
}
Expand All @@ -427,8 +437,10 @@ func (s3 *S3) listRepos() (ret []*Repo, err error) {

func (s3 *S3) statFile(key string) (info *objectInfo, err error) {
svc := s3.getService()
ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second)
defer cancelFn()

header, err := svc.HeadObject(&as3.HeadObjectInput{
header, err := svc.HeadObject(ctx, &as3.HeadObjectInput{
Bucket: &s3.Conf.S3.Bucket,
Key: &key,
})
Expand Down Expand Up @@ -479,27 +491,36 @@ func (s3 *S3) getNotFound(keys []string) (ret []string, err error) {
return
}

func (s3 *S3) getService() *as3.S3 {
sess := session.Must(session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s3.Conf.S3.AccessKey, s3.Conf.S3.SecretKey, ""),
Endpoint: aws.String(s3.Conf.S3.Endpoint),
Region: aws.String(s3.Conf.S3.Region),
S3ForcePathStyle: aws.Bool(s3.Conf.S3.PathStyle),
HTTPClient: s3.HTTPClient,
}))
return as3.New(sess)
func (s3 *S3) getService() *as3.Client {
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
logging.LogErrorf("load default config failed: %s", err)
}

return as3.NewFromConfig(cfg, func(o *as3.Options) {
o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(s3.Conf.S3.AccessKey, s3.Conf.S3.SecretKey, ""))
o.BaseEndpoint = aws.String(s3.Conf.S3.Endpoint)
o.Region = s3.Conf.S3.Region
o.UsePathStyle = s3.Conf.S3.PathStyle
o.HTTPClient = s3.HTTPClient
})
}

func (s3 *S3) isErrNotFound(err error) bool {
switch err.(type) {
case awserr.Error:
code := err.(awserr.Error).Code()
switch code {
case as3.ErrCodeNoSuchKey:
return true
}
var nsk *as3Types.NoSuchKey
if errors.As(err, &nsk) {
return true
}

var nf *as3Types.NotFound
if errors.As(err, &nf) {
return true
}

msg := strings.ToLower(err.Error())
return strings.Contains(msg, "does not exist") || strings.Contains(msg, "404") || strings.Contains(msg, "no such file or directory")
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
msg := strings.ToLower(apiErr.ErrorMessage())
return strings.Contains(msg, "does not exist") || strings.Contains(msg, "404") || strings.Contains(msg, "no such file or directory")
}
return false
}
21 changes: 18 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ toolchain go1.23.1
require (
github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7
github.com/88250/gulu v1.2.3-0.20241212012748-c4dc08fe45ec
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go-v2 v1.32.7
github.com/aws/aws-sdk-go-v2/config v1.28.7
github.com/aws/aws-sdk-go-v2/credentials v1.17.48
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1
github.com/aws/smithy-go v1.22.1
github.com/dgraph-io/ristretto v1.0.0
github.com/klauspost/compress v1.17.11
github.com/panjf2000/ants/v2 v2.10.0
Expand All @@ -25,6 +29,19 @@ require (
require (
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
Expand All @@ -34,7 +51,6 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/imroc/req/v3 v3.49.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/onsi/ginkgo/v2 v2.22.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
Expand All @@ -49,7 +65,6 @@ require (
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/tools v0.28.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

//replace github.com/siyuan-note/filelock => D:\88250\filelock
45 changes: 36 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,42 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE=
github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M=
github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ=
github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
Expand Down Expand Up @@ -48,10 +82,6 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/imroc/req/v3 v3.49.0 h1:5Rac2qvz7Dq0E3PeBo/c2szV3hagPQIGLoHtfBmYhu4=
github.com/imroc/req/v3 v3.49.0/go.mod h1:XZf4t94DNJzcA0UOBlA68hmSrWsAyvN407ADdH4mzCA=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
Expand Down Expand Up @@ -186,9 +216,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down

0 comments on commit b6410d9

Please sign in to comment.