Skip to content

Commit

Permalink
✨ feat(storage): Add S3 storage (#368)
Browse files Browse the repository at this point in the history
  • Loading branch information
thlz998 authored Sep 27, 2024
1 parent f58fe0b commit 226864f
Show file tree
Hide file tree
Showing 5 changed files with 127 additions and 5 deletions.
82 changes: 82 additions & 0 deletions common/storage/drives/s3.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package drives

import (
"bytes"
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)

type S3Upload struct {
EndPoint string
CustomDomain string
AccessKeyId string
AccessKeySecret string
BucketName string
}

func NewS3Upload(endpoint, accessKeyId, accessKeySecret, bucketName, cdnurl string) *S3Upload {
_cdnurl := cdnurl
if _cdnurl == "" {
_cdnurl = endpoint
}
return &S3Upload{
EndPoint: endpoint,
BucketName: bucketName,
CustomDomain: _cdnurl,
AccessKeyId: accessKeyId,
AccessKeySecret: accessKeySecret,
}
}

func (a *S3Upload) Name() string {
return "S3"
}

func (a *S3Upload) Upload(data []byte, s3Key string) (string, error) {

// 创建 S3 会话
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(
a.AccessKeyId,
a.AccessKeySecret,
"",
),
Endpoint: aws.String(a.EndPoint),
Region: aws.String("auto"),
S3ForcePathStyle: aws.Bool(true),
})
if err != nil {
return "", fmt.Errorf("failed to create session: %v", err)
}

svc := s3.New(sess)

// 检查文件是否已存在于 S3
_, err = svc.HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(a.BucketName),
Key: aws.String(s3Key),
})

if err == nil {
// 文件已存在,直接返回自定义域名 URL
return fmt.Sprintf("%s/%s", a.CustomDomain, s3Key), nil
}
fileBytes := bytes.NewReader(data)

// 上传文件到 S3
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(a.BucketName),
Key: aws.String(s3Key),
Body: fileBytes,
})

if err != nil {
return "", fmt.Errorf("failed to upload file to S3: %v", err)
}

return fmt.Sprintf("%s/%s", a.CustomDomain, s3Key), nil
}
27 changes: 27 additions & 0 deletions common/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ func InitStorage() {
InitImgurStorage()
InitSMStorage()
InitALIOSSStorage()
InitS3Storage()
}

func InitALIOSSStorage() {
Expand Down Expand Up @@ -58,3 +59,29 @@ func InitImgurStorage() {
imgurUpload := drives.NewImgurUpload(imgurClientId)
AddStorageDrive(imgurUpload)
}

func InitS3Storage() {
endpoint := viper.GetString("storage.s3.endpoint")
if endpoint == "" {
return
}
accessKeyId := viper.GetString("storage.s3.accessKeyId")
if accessKeyId == "" {
return
}
accessKeySecret := viper.GetString("storage.s3.accessKeySecret")
if accessKeySecret == "" {
return
}
bucketName := viper.GetString("storage.s3.bucketName")
if bucketName == "" {
return
}
cdnurl := viper.GetString("storage.s3.cdnurl")
if cdnurl == "" {
cdnurl = endpoint
}

s3Upload := drives.NewS3Upload(endpoint, accessKeyId, accessKeySecret, bucketName, cdnurl)
AddStorageDrive(s3Upload)
}
8 changes: 7 additions & 1 deletion config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,4 +81,10 @@ storage: # 存储设置 (可选,主要用于图片生成,有些供应商不提
endpoint: "" # Endpoint(地域节点),比如oss-cn-beijing.aliyuncs.com
bucketName: "" # Bucket名称,比如zerodeng-superai
accessKeyId: "" # 阿里授权KEY,在阿里云后台用户RAM控制部分获取
accessKeySecret: "" # 阿里授权SECRET,在阿里云后台用户RAM控制部分获取
accessKeySecret: "" # 阿里授权SECRET,在阿里云后台用户RAM控制部分获取
s3: # AwsS3协议
endpoint: "" # Endpoint(地域节点),比如https://xxxxxx.r2.cloudflarestorage.com
cdnurl: "" # 公共访问域名,比如https://pub-xxxxx.r2.dev,如果不配置则使用endpoint
bucketName: "" # Bucket名称,比如zerodeng-superai
accessKeyId: "" # accessKeyId
accessKeySecret: "" # accessKeySecret
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ toolchain go1.22.3
require (
cloud.google.com/go/iam v1.1.11
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1
github.com/aws/smithy-go v1.20.1
github.com/bwmarrin/snowflake v0.3.0
github.com/coocood/freecache v1.2.4
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/eko/gocache/lib/v4 v4.1.6
github.com/eko/gocache/store/freecache/v4 v4.2.2
github.com/eko/gocache/store/redis/v4 v4.2.2
Expand Down Expand Up @@ -40,6 +42,7 @@ require (
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.25.0
golang.org/x/image v0.18.0
golang.org/x/oauth2 v0.23.0
google.golang.org/api v0.188.0
google.golang.org/grpc v1.64.1
gopkg.in/natefinch/lumberjack.v2 v2.2.1
Expand All @@ -56,7 +59,6 @@ require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
github.com/coreos/go-oidc v2.2.1+incompatible // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
Expand All @@ -69,6 +71,7 @@ require (
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
Expand Down Expand Up @@ -98,7 +101,6 @@ require (
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
Expand Down
9 changes: 7 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/antonlindstrom/pgstore v0.0.0-20200229204646-b08ebf1105e0/go.mod h1:2Ti6VUHVxpC0VSmTZzEvpzysnaGAfGBOoMIz5ykPyyw=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
Expand Down Expand Up @@ -291,6 +293,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
Expand Down Expand Up @@ -580,8 +586,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -811,6 +815,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down

0 comments on commit 226864f

Please sign in to comment.