From b3fe040c5377aea0779282dcfe53e39aa4bc58a5 Mon Sep 17 00:00:00 2001 From: handlename Date: Thu, 16 May 2024 17:03:40 +0900 Subject: [PATCH 1/8] feat: use bucket region for s3 client --- go.mod | 30 +++++++++++++------------ go.sum | 67 ++++++++++++++++++++++++++++++++----------------------- router.go | 38 ++++++++++++++++++++++++++++--- 3 files changed, 90 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index f069b77..4217fd3 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.22 require ( github.com/aws/aws-lambda-go v1.47.0 - github.com/aws/aws-sdk-go-v2 v1.26.1 - github.com/aws/aws-sdk-go-v2/config v1.27.11 - github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/aws-sdk-go-v2 v1.26.2 + github.com/aws/aws-sdk-go-v2/config v1.27.14 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.19 + github.com/aws/aws-sdk-go-v2/service/s3 v1.54.1 github.com/google/go-cmp v0.6.0 github.com/mickep76/mapslice-json v0.0.0-20200219143743-9f118f7dce45 github.com/pkg/errors v0.9.1 @@ -15,18 +16,19 @@ require ( require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.6 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.8 // indirect github.com/aws/smithy-go v1.20.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect ) diff --git a/go.sum b/go.sum index ddca511..6ee4a86 100644 --- a/go.sum +++ b/go.sum @@ -1,54 +1,65 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= -github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= -github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2 v1.26.2 h1:OTRAL8EPdNoOdiq5SUhCaHhVPBU2wxAUe5uwasoJGRM= +github.com/aws/aws-sdk-go-v2 v1.26.2/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= -github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= -github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= -github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/config v1.27.14 h1:QOg8Ud53rrmdjBHX080AaYUBhG2ER28kP/yjE7afF/0= +github.com/aws/aws-sdk-go-v2/config v1.27.14/go.mod h1:CLgU27opbIwnjwH++zQPvF4qsEIqviKL6l8b1AtRImc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.14 h1:0y1IAEldTO2ZA3Lcq7u7y4Q2tUQlB3At2LZQijUHu3U= +github.com/aws/aws-sdk-go-v2/credentials v1.17.14/go.mod h1:En2zXCfDZJgtbp2UnzHDgKMz+mSRc4pA3Ka+jxoJvaA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.2 h1:HTAQSEibYaSioHzjOQssUJnE8itwVP9SzmdR6lqC38g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.2/go.mod h1:NjUtmUEIimOc5tPw//xqKNK/spUqCTSbxjwzCrnsj8U= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.19 h1:5ez5wz0csAnSMhR98GLthsU/5V5YkKbPm3xWI8R578Y= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.19/go.mod h1:eczzjxP9gnNAXisg4tn+a8paeE5nO1WSGFnsFqHFcLk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.6 h1:yrfbQyxO73opeqep8FohU4LJx56iiQuvf4/XPgFB4To= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.6/go.mod h1:bFtlRACYBPG2AUYst0ky5TPtgeYqWCksozVTGsZ1zq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.6 h1:DXsuqiAp1mGkelZCUSex8DsRtkeK4mW3oreyjNSegoo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.6/go.mod h1:cLtGzsyh+Wz2j1w9Qyfn5DA9i25RfbYjwfJBZqCiP9Y= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.6 h1:+/uB/M07Isd7UajQIYW2M4lDc/302gIWu1zMe0d7uKo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.6/go.mod h1:7Gw/GeEezsEzpU/f1JWzSb1Y4M05taehNadic8jfF8U= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.8 h1:Etk3MkNNAXLKkOkrLcacOZhbH+ICPpM05Wueg/4FZGw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.8/go.mod h1:LwdzRBRLEOcMRhcM31ltkzl8Nw4XpnmBkOksr+LY53A= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.8 h1:gwdGHxiV5f6Of48JJIZVD7sx45kT1l9kYdoUH5oQTZM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.8/go.mod h1:C9Glc6N50uIJqPPeL6N3spW/wzGyeQsQmecnKS7DTR4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.6 h1:T9S8fTr0gppdTU6rjrSBl9VFiabxUw41dqzUSozuQP8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.6/go.mod h1:mQUaFeeWztcGGBaOxSykT9H/qF+FGw3sCGERh3APsRc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.54.1 h1:q8hQJhndXQXnCnd4nG608lfhapDvylff9CCQyWX0tHQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.54.1/go.mod h1:4QIYvQFYGX6Q2qhLW6KeJO0iXQUFJ/WLBL0pSdazHwA= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.7 h1:sdPpNCoUijc0ntu024ZdjrXh3mB9rud5SjmE7djIfK4= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.7/go.mod h1:8RMeDMFTkkDQ5LvaaAykdkNVVR0eQxGWm8CD6uBvd1M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.1 h1:/vljM1ZswUEIRHWVxEqDhLzOSGmDcstW2zeTt23Ipf0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.1/go.mod h1:XhJksmKh1RYjMbWHf3ZwQF0UYJjlqrm45NVvDe54SOU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.8 h1:FCYhQETaff4Skb2Hz9WoUqJAesr4MIQ9+TQ9ypjz7Ic= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.8/go.mod h1:s+7oFIwiOegfrF00xNowWwLAtRiA9xhvm1UpZdJ0aus= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/mickep76/mapslice-json v0.0.0-20200219143743-9f118f7dce45 h1:qV4L2O3zhoPwDlk7QZYMhSYbo05aHt9uyzzsi/BiUOM= github.com/mickep76/mapslice-json v0.0.0-20200219143743-9f118f7dce45/go.mod h1:Fpzmz4najGi/+LKF7hjt/SpVA5044oZ8RFt+AAgyu2Q= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/router.go b/router.go index 2f822ee..37df82c 100644 --- a/router.go +++ b/router.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" @@ -209,7 +210,12 @@ func (r *Router) getS3Object(ctx context.Context, s3url string) (io.ReadCloser, return nil, errors.New("s3:// required") } - out, err := r.s3.GetObject(ctx, &s3.GetObjectInput{ + s3c, err := r.getS3Client(ctx, u.Host) + if err != nil { + return nil, err + } + + out, err := s3c.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(u.Host), Key: aws.String(strings.TrimPrefix(u.Path, "/")), }) @@ -246,6 +252,11 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek r.sem.Acquire(ctx, 1) defer r.sem.Release(1) + s3c, err := r.getS3Client(ctx, dest.Bucket) + if err != nil { + return err + } + in := &s3.PutObjectInput{ Bucket: &dest.Bucket, Key: &dest.Key, @@ -253,13 +264,34 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek Metadata: meta, } log.Println("[info] starting put to", dest.String()) - _, err := r.s3.PutObject(ctx, in) - if err == nil { + if _, err := s3c.PutObject(ctx, in); err != nil { log.Println("[info] completed put to", dest.String()) } return err } +// getS3Client returns s3 client for same region as the bucket +func (r *Router) getS3Client(ctx context.Context, bucket string) (*s3.Client, error) { + bucketRegion, err := manager.GetBucketRegion(ctx, r.s3, bucket) + if err != nil { + return nil, err + } + + if r.s3.Options().Region == bucketRegion { + return r.s3, nil + } + + awsConfig, err := config.LoadDefaultConfig( + ctx, + config.WithRegion(bucketRegion), + ) + if err != nil { + return nil, err + } + + return s3.NewFromConfig(awsConfig), nil +} + type record struct { parsed map[string]interface{} raw []byte From 1af7a297cdc134a70bcf8bad1499b94781ad2e02 Mon Sep 17 00:00:00 2001 From: handlename Date: Thu, 16 May 2024 17:06:07 +0900 Subject: [PATCH 2/8] fix: err condition --- router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router.go b/router.go index 37df82c..43b33ad 100644 --- a/router.go +++ b/router.go @@ -264,7 +264,7 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek Metadata: meta, } log.Println("[info] starting put to", dest.String()) - if _, err := s3c.PutObject(ctx, in); err != nil { + if _, err := s3c.PutObject(ctx, in); err == nil { log.Println("[info] completed put to", dest.String()) } return err From 8d2ac37e13b79f28c7bedfa3121358b26ac854ce Mon Sep 17 00:00:00 2001 From: handlename Date: Fri, 17 May 2024 11:35:04 +0900 Subject: [PATCH 3/8] feat: cache S3 client for each regions --- router.go | 64 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/router.go b/router.go index 43b33ad..d11ebba 100644 --- a/router.go +++ b/router.go @@ -40,7 +40,14 @@ var ( // Router represents s3-object-router application type Router struct { - s3 *s3.Client + awsConf aws.Config + + // s3 clients for each region + s3 map[string]*s3.Client + + // s3 bucket region cache + s3bucketRegion map[string]string + option *Option sem *semaphore.Weighted @@ -67,9 +74,13 @@ func New(opt *Option) (*Router, error) { } return &Router{ - s3: s3.NewFromConfig(awsConf), - option: opt, - sem: semaphore.NewWeighted(int64(MaxConcurrency)), + awsConf: awsConf, + s3: map[string]*s3.Client{ + awsConf.Region: s3.NewFromConfig(awsConf), + }, + s3bucketRegion: map[string]string{}, + option: opt, + sem: semaphore.NewWeighted(int64(MaxConcurrency)), genKeyPrefix: func(r *record) (string, error) { var b strings.Builder if err := tmpl.Execute(&b, r.parsed); err != nil { @@ -210,7 +221,7 @@ func (r *Router) getS3Object(ctx context.Context, s3url string) (io.ReadCloser, return nil, errors.New("s3:// required") } - s3c, err := r.getS3Client(ctx, u.Host) + s3c, err := r.s3Client(ctx, u.Host) if err != nil { return nil, err } @@ -252,7 +263,7 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek r.sem.Acquire(ctx, 1) defer r.sem.Release(1) - s3c, err := r.getS3Client(ctx, dest.Bucket) + s3c, err := r.s3Client(ctx, dest.Bucket) if err != nil { return err } @@ -270,26 +281,45 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek return err } -// getS3Client returns s3 client for same region as the bucket -func (r *Router) getS3Client(ctx context.Context, bucket string) (*s3.Client, error) { - bucketRegion, err := manager.GetBucketRegion(ctx, r.s3, bucket) +func (r *Router) defaultS3Client() *s3.Client { + return r.s3[r.awsConf.Region] +} + +// s3Client returns s3 client for same region as the bucket +func (r *Router) s3Client(ctx context.Context, bucket string) (*s3.Client, error) { + bucketRegion, err := r.getS3BucketRegion(ctx, bucket) if err != nil { return nil, err } - if r.s3.Options().Region == bucketRegion { - return r.s3, nil + if s3, ok := r.s3[bucketRegion]; ok { + return s3, nil } - awsConfig, err := config.LoadDefaultConfig( - ctx, - config.WithRegion(bucketRegion), - ) + awsConfig := r.awsConf.Copy() + awsConfig.Region = bucketRegion + s3 := s3.NewFromConfig(awsConfig) + r.s3[bucketRegion] = s3 + + return s3, nil +} + +func (r *Router) getS3BucketRegion(ctx context.Context, bucket string) (string, error) { + log.Println("in getS3BucketRegion") + if region, ok := r.s3bucketRegion[bucket]; ok { + log.Printf("[debug] bucket region for %s is cached: %s\n", bucket, region) + return region, nil + } + + region, err := manager.GetBucketRegion(ctx, r.defaultS3Client(), bucket) if err != nil { - return nil, err + return "", err } - return s3.NewFromConfig(awsConfig), nil + r.s3bucketRegion[bucket] = region + log.Printf("[debug] bucket region for %s is %s, added to cache\n", bucket, region) + + return region, nil } type record struct { From a28fb98ba9df11f2215dde99803682d7f15a2841 Mon Sep 17 00:00:00 2001 From: handlename Date: Fri, 17 May 2024 14:04:12 +0900 Subject: [PATCH 4/8] refactor: remove debug message --- router.go | 1 - 1 file changed, 1 deletion(-) diff --git a/router.go b/router.go index d11ebba..cfa43fa 100644 --- a/router.go +++ b/router.go @@ -305,7 +305,6 @@ func (r *Router) s3Client(ctx context.Context, bucket string) (*s3.Client, error } func (r *Router) getS3BucketRegion(ctx context.Context, bucket string) (string, error) { - log.Println("in getS3BucketRegion") if region, ok := r.s3bucketRegion[bucket]; ok { log.Printf("[debug] bucket region for %s is cached: %s\n", bucket, region) return region, nil From 189c1190f06f7090693a01a3ceaf3b1a216c572f Mon Sep 17 00:00:00 2001 From: handlename Date: Fri, 17 May 2024 16:05:10 +0900 Subject: [PATCH 5/8] fix: exclusive use of maps --- router.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/router.go b/router.go index cfa43fa..4b9549d 100644 --- a/router.go +++ b/router.go @@ -14,6 +14,7 @@ import ( "net/url" "path" "strings" + "sync" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -44,9 +45,11 @@ type Router struct { // s3 clients for each region s3 map[string]*s3.Client + s3Lock sync.Mutex // s3 bucket region cache s3bucketRegion map[string]string + s3bucketRegionLock sync.Mutex option *Option sem *semaphore.Weighted @@ -292,6 +295,9 @@ func (r *Router) s3Client(ctx context.Context, bucket string) (*s3.Client, error return nil, err } + r.s3Lock.Lock() + r.s3Lock.Unlock() + if s3, ok := r.s3[bucketRegion]; ok { return s3, nil } @@ -305,6 +311,9 @@ func (r *Router) s3Client(ctx context.Context, bucket string) (*s3.Client, error } func (r *Router) getS3BucketRegion(ctx context.Context, bucket string) (string, error) { + r.s3bucketRegionLock.Lock() + defer r.s3bucketRegionLock.Unlock() + if region, ok := r.s3bucketRegion[bucket]; ok { log.Printf("[debug] bucket region for %s is cached: %s\n", bucket, region) return region, nil From 81184fd9f641197557bb3a51fc2ec9383cf79b10 Mon Sep 17 00:00:00 2001 From: handlename Date: Fri, 17 May 2024 16:06:29 +0900 Subject: [PATCH 6/8] style: gofmt --- router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router.go b/router.go index 4b9549d..9f4339f 100644 --- a/router.go +++ b/router.go @@ -44,11 +44,11 @@ type Router struct { awsConf aws.Config // s3 clients for each region - s3 map[string]*s3.Client + s3 map[string]*s3.Client s3Lock sync.Mutex // s3 bucket region cache - s3bucketRegion map[string]string + s3bucketRegion map[string]string s3bucketRegionLock sync.Mutex option *Option From 14095549fc9fc582213e01a203ef69c401220b16 Mon Sep 17 00:00:00 2001 From: NAGATA Hiroaki Date: Tue, 21 May 2024 07:31:29 +0900 Subject: [PATCH 7/8] fix: missing defer Co-authored-by: FUJIWARA Shunichiro --- router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router.go b/router.go index 9f4339f..4e5c091 100644 --- a/router.go +++ b/router.go @@ -296,7 +296,7 @@ func (r *Router) s3Client(ctx context.Context, bucket string) (*s3.Client, error } r.s3Lock.Lock() - r.s3Lock.Unlock() + defer r.s3Lock.Unlock() if s3, ok := r.s3[bucketRegion]; ok { return s3, nil From 5620d9960756e57532e098baf73520abb9f78f5c Mon Sep 17 00:00:00 2001 From: handlename Date: Wed, 22 May 2024 14:23:24 +0900 Subject: [PATCH 8/8] fix: exclusive use of map missed on 189c1190f06f7090693a01a3ceaf3b1a216c572f --- router.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/router.go b/router.go index 4e5c091..f7609fd 100644 --- a/router.go +++ b/router.go @@ -285,6 +285,8 @@ func (r *Router) putToS3(ctx context.Context, dest destination, body io.ReadSeek } func (r *Router) defaultS3Client() *s3.Client { + r.s3Lock.Lock() + defer r.s3Lock.Unlock() return r.s3[r.awsConf.Region] }