From a56f0b8150a00ad1d3faf73d8485593ff42fd77a Mon Sep 17 00:00:00 2001 From: neo-f Date: Thu, 23 Mar 2023 13:52:44 +0800 Subject: [PATCH] feat(storage): add cos support --- go.mod | 4 ++ go.sum | 15 +++++ internal/config/config.go | 33 ---------- internal/config/{const.go => static.go} | 4 ++ internal/dal/model/docs.go | 6 +- internal/dal/oss.go | 50 --------------- internal/dal/storage.go | 31 ++++++++++ internal/router/doc.go | 8 +++ internal/schema/docs.go | 15 ++++- pkg/s3/s3.go | 1 - pkg/storage/storage.go | 44 +++++++++++++ pkg/storage/storage_cos.go | 82 +++++++++++++++++++++++++ pkg/storage/storage_minio.go | 80 ++++++++++++++++++++++++ 13 files changed, 283 insertions(+), 90 deletions(-) rename internal/config/{const.go => static.go} (89%) delete mode 100644 internal/dal/oss.go create mode 100644 internal/dal/storage.go delete mode 100644 pkg/s3/s3.go create mode 100644 pkg/storage/storage.go create mode 100644 pkg/storage/storage_cos.go create mode 100644 pkg/storage/storage_minio.go diff --git a/go.mod b/go.mod index 15aa106..551a75b 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 + github.com/tencentyun/cos-go-sdk-v5 v0.7.41 github.com/tidwall/gjson v1.14.4 github.com/tidwall/sjson v1.2.5 gorm.io/driver/postgres v1.5.0 @@ -27,6 +28,7 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/clbanning/mxj v1.8.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -36,6 +38,7 @@ require ( github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/schema v1.2.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -64,6 +67,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/mozillazg/go-httpheader v0.3.1 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/perimeterx/marshmallow v1.1.4 // indirect github.com/philhofer/fwd v1.1.1 // indirect diff --git a/go.sum b/go.sum index 9b4f5f2..b4bf928 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -58,6 +59,8 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -159,6 +162,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -174,6 +180,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -265,6 +272,7 @@ github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMv github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -276,6 +284,9 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok= +github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/neo-f/soda v0.2.6 h1:F5AlSOTwNiS1te9muEHAyQc1CwWw9IV22K3TsDsL7vU= @@ -368,6 +379,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= +github.com/tencentyun/cos-go-sdk-v5 v0.7.41 h1:iU0Li/Np78H4SBna0ECQoF3mpgi6ImLXU+doGzPFXGc= +github.com/tencentyun/cos-go-sdk-v5 v0.7.41/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/internal/config/config.go b/internal/config/config.go index 6f9efe1..f6e1bff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,6 @@ package config import ( - "net/url" "os" "path/filepath" "runtime" @@ -11,14 +10,9 @@ import ( "github.com/go-playground/validator" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/spf13/cast" "github.com/spf13/viper" ) -var ENVS = [][2]string{ - {"本地测试环境", "http://localhost:8080"}, -} - type Config struct { IsLocal bool Debug bool `mapstructure:"debug"` @@ -36,33 +30,6 @@ type Config struct { } } -type ossConfig struct { - Schema string - Endpoint string - AccessID string - AccessSecret string - Region string - Bucket string - Secure bool -} - -func (c *Config) GetOSSConfig() *ossConfig { - oss, err := url.Parse(c.Databases.OSS) - if err != nil { - log.Fatal().Err(err).Msg("parse oss config error") - } - accessSecret, _ := oss.User.Password() - return &ossConfig{ - Schema: oss.Scheme, - Endpoint: oss.Host, - AccessID: oss.User.Username(), - AccessSecret: accessSecret, - Region: oss.Query().Get("region"), - Bucket: oss.Query().Get("bucket"), - Secure: cast.ToBool(oss.Query().Get("secure")), - } -} - var ( c *Config once sync.Once diff --git a/internal/config/const.go b/internal/config/static.go similarity index 89% rename from internal/config/const.go rename to internal/config/static.go index 18323ac..e175667 100644 --- a/internal/config/const.go +++ b/internal/config/static.go @@ -23,3 +23,7 @@ const ( LogTagStaffID = "staff_id" LogTagTraceID = "trace_id" ) + +var ENVS = [][2]string{ + {"本地测试环境", "http://localhost:8080"}, +} diff --git a/internal/dal/model/docs.go b/internal/dal/model/docs.go index ee8f006..9a828e6 100644 --- a/internal/dal/model/docs.go +++ b/internal/dal/model/docs.go @@ -3,7 +3,6 @@ package model import ( "context" "net/url" - "octopus/internal/config" "octopus/internal/dal" "time" @@ -37,15 +36,14 @@ type Doc struct { IsDeletable bool `gorm:"column:is_deletable;type:boolean;not null;default:true"` // 是否允许被删除 IsEditable bool `gorm:"column:is_editable;type:boolean;not null;default:true"` // 是否允许编辑名称 FolderID string `gorm:"column:folder_id;type:varchar;index:idx_folder_id"` // 文件夹ID - OSSObjectID string `gorm:"column:oss_object_id;type:varchar"` // OSS Object ID + ObjectName string `gorm:"column:object_name;type:varchar"` // 对象存储中对应的object_name CreatedBy string `gorm:"column:created_by;type:varchar;not null"` // 创建人 Folder *DocFolder `gorm:"foreignKey:FolderID;references:ID"` } func (df *Doc) PresignedURL(ctx context.Context) (*url.URL, error) { - bucket := config.Get().GetOSSConfig().Bucket - return dal.GetMinio().PresignedGetObject(ctx, bucket, df.OSSObjectID, time.Hour, nil) + return dal.GetStorage().PresignedGetObject(ctx, df.ObjectName, time.Hour) } func (*Doc) TableName() string { diff --git a/internal/dal/oss.go b/internal/dal/oss.go deleted file mode 100644 index 857bacd..0000000 --- a/internal/dal/oss.go +++ /dev/null @@ -1,50 +0,0 @@ -package dal - -import ( - "net/url" - "octopus/internal/config" - "sync" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/rs/zerolog/log" - // pb "github.com/qdrant/go-client/qdrant" -) - -var ( - ossOnce sync.Once - ossInstance *minio.Client -) - -func GetMinio() *minio.Client { - ossOnce.Do(initMinio) - return ossInstance -} - -func initMinio() { - log.Info().Msg("loading minio configs") - - ossConfig, err := url.Parse(config.Get().Databases.OSS) - if err != nil { - log.Fatal().Err(err).Msg("parse oss config error") - } - accessSecret, _ := ossConfig.User.Password() - // defaultConfig := &model.StorageConfig{ - // Schema: ossConfig.Scheme, - // Endpoint: ossConfig.Host, - // AccessID: ossConfig.User.Username(), - // AccessSecret: accessSecret, - // Bucket: ossConfig.Query().Get("bucket"), - // Region: ossConfig.Query().Get("region"), - // Secure: cast.ToBool(ossConfig.Query().Get("secure")), - // } - minioClient, err := minio.New(ossConfig.Host, &minio.Options{ - Creds: credentials.NewStaticV4(ossConfig.User.Username(), accessSecret, ""), - Secure: false, - }) - if err != nil { - log.Fatal().Msgf("minio client init error: %v", err) - } - - ossInstance = minioClient -} diff --git a/internal/dal/storage.go b/internal/dal/storage.go new file mode 100644 index 0000000..2104547 --- /dev/null +++ b/internal/dal/storage.go @@ -0,0 +1,31 @@ +package dal + +import ( + "octopus/internal/config" + "octopus/pkg/storage" + "sync" + + "github.com/rs/zerolog/log" + // pb "github.com/qdrant/go-client/qdrant" +) + +var ( + storageOnce sync.Once + storageInstance storage.ObjectStorage +) + +func GetStorage() storage.ObjectStorage { + storageOnce.Do(initMinio) + return storageInstance +} + +func initMinio() { + log.Info().Msg("loading minio configs") + + s, err := storage.NewObjectStorage(config.Get().Databases.OSS) + if err != nil { + log.Fatal().Msgf("storage client init failed: %v", err) + } + + storageInstance = s +} diff --git a/internal/router/doc.go b/internal/router/doc.go index e16a493..9cae4fd 100644 --- a/internal/router/doc.go +++ b/internal/router/doc.go @@ -29,6 +29,14 @@ func RegisterDocRouter(app *soda.Soda) { SetJSONRequestBody(schema.UpdateDoc{}). AddJSONResponse(200, schema.Doc{}).OK() + // get presigned url for tmp file upload + app.Get("/docs/upload-url", nil). + AddTags("文档管理"). + SetSummary("获取临时上传文件用的预签名URL"). + AddJWTSecurity(JWTRequired). + SetParameters(schema.CreateUploadURL{}). + AddJSONResponse(200, schema.UploadURL{}).OK() + app.Delete("/docs/:id", nil). AddTags("文档管理"). SetSummary("获取文档列表"). diff --git a/internal/schema/docs.go b/internal/schema/docs.go index 91dc692..403f311 100644 --- a/internal/schema/docs.go +++ b/internal/schema/docs.go @@ -1,6 +1,7 @@ package schema import ( + "net/url" "octopus/pkg/tools" "time" @@ -63,8 +64,9 @@ type DocID struct { } type CreateDoc struct { - Name string `json:"name" validate:"required" oai:"description=文档名称"` - FolderID string `json:"folder_id" validate:"required" oai:"description=归属文件夹ID"` + Name string `json:"name" validate:"required" oai:"description=文档名称"` + ObjectName string `json:"object_name" validate:"required" oai:"description=对象存储中对应的文档对象名称"` + FolderID string `json:"folder_id" validate:"required" oai:"description=归属文件夹ID"` } type UpdateDoc struct { @@ -104,3 +106,12 @@ func (q *ListDocQuery) Validate() error { } return nil } + +type CreateUploadURL struct { + FileName string `query:"file_name" validate:"required" oai:"description=上传文件名"` +} + +type UploadURL struct { + URL url.URL `json:"url" oai:"description=生成的预签名上传URL"` + ObjectName string `json:"object_name" oai:"description=上传文件名"` +} diff --git a/pkg/s3/s3.go b/pkg/s3/s3.go deleted file mode 100644 index 3ed7f97..0000000 --- a/pkg/s3/s3.go +++ /dev/null @@ -1 +0,0 @@ -package s3 diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..b95b8bf --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,44 @@ +package storage + +import ( + "context" + "errors" + "net/url" + "time" + + "github.com/spf13/cast" +) + +type ObjectStorage interface { + // 获取上传对象预签名URL + PresignedPutObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) + // 获取访问对象预签名URL + PresignedGetObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) + // 移动对象 + MoveObject(ctx context.Context, src, dst string) error + // 检查文件是否存在 + ObjectExists(ctx context.Context, objectName string) (bool, error) +} + +func NewObjectStorage(dsn string) (ObjectStorage, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, err + } + + user := u.User.Username() + pass, _ := u.User.Password() + + switch u.Scheme { + case "cos": + // ex: cos://examplebucket-1250000000.cos.COS_REGION.myqcloud.com + return newCosStorage(u.Host, user, pass) + case "minio": + // ex: minio://minio-api:9001?secure=false&bucket=images + secure := cast.ToBool(u.Query().Get("secure")) + bucket := u.Query().Get("bucket") + return newMinioStorage(u.Host, user, pass, secure, bucket) + default: + return nil, errors.New("the storage type not support yet") + } +} diff --git a/pkg/storage/storage_cos.go b/pkg/storage/storage_cos.go new file mode 100644 index 0000000..51f2a9e --- /dev/null +++ b/pkg/storage/storage_cos.go @@ -0,0 +1,82 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +var _ ObjectStorage = (*storageCos)(nil) + +type storageCos struct { + client *cos.Client + + host string + accessKey string + accessSecret string + bucket string +} + +func newCosStorage(host, accessKey, accessSecret string) (ObjectStorage, error) { + parts := strings.Split(host, ".") + if len(parts) != 5 { + return nil, errors.New("invalid cos host") + } + parts = strings.Split(parts[0], "-") + if len(parts) != 2 { + return nil, errors.New("invalid cos host") + } + bucketName := parts[0] + + u, err := url.Parse("https://" + host) + if err != nil { + return nil, err + } + client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: accessKey, + SecretKey: accessSecret, + }, + }) + return &storageCos{ + client: client, + host: host, + accessKey: accessKey, + accessSecret: accessSecret, + bucket: bucketName, + }, nil +} + +// PresignedPutObject implements ObjectStorage +func (c *storageCos) PresignedPutObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) { + return c.client.Object.GetPresignedURL(ctx, "PUT", objectName, c.accessKey, c.accessSecret, expires, nil) +} + +// PresignedGetObject implements ObjectStorage +func (c *storageCos) PresignedGetObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) { + return c.client.Object.GetPresignedURL(ctx, "GET", objectName, c.accessKey, c.accessSecret, expires, nil) +} + +// MoveObject implements ObjectStorage +func (c *storageCos) MoveObject(ctx context.Context, src string, dst string) error { + srcURL := fmt.Sprintf("%s/%s", c.host, src) + + if _, _, err := c.client.Object.Copy(ctx, dst, srcURL, nil); err != nil { + return fmt.Errorf("copy object failed while move object: %w", err) + } + if _, err := c.client.Object.Delete(ctx, src, nil); err != nil { + return fmt.Errorf("delete object failed while move object: %w", err) + } + return nil +} + +// ObjectExists implements ObjectStorage +func (c *storageCos) ObjectExists(ctx context.Context, objectName string) (bool, error) { + return c.client.Object.IsExist(ctx, objectName) +} diff --git a/pkg/storage/storage_minio.go b/pkg/storage/storage_minio.go new file mode 100644 index 0000000..c435a2d --- /dev/null +++ b/pkg/storage/storage_minio.go @@ -0,0 +1,80 @@ +package storage + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var _ ObjectStorage = (*storageMinio)(nil) + +type storageMinio struct { + client *minio.Client + + endpoint string + accessKey string + accessSecret string + secure bool + + bucket string +} + +func newMinioStorage(endpoint, accessKey, accessSecret string, secure bool, bucket string) (ObjectStorage, error) { + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, accessSecret, ""), + Secure: secure, + }) + if err != nil { + return nil, err + } + return &storageMinio{ + client: client, + endpoint: endpoint, + accessKey: accessKey, + accessSecret: accessSecret, + secure: secure, + bucket: bucket, + }, nil +} + +// PresignedPutObject implements ObjectStorage +func (m *storageMinio) PresignedPutObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) { + return m.client.PresignedPutObject(ctx, m.bucket, objectName, expires) +} + +// PresignedGetObject implements ObjectStorage +func (m *storageMinio) PresignedGetObject(ctx context.Context, objectName string, expires time.Duration) (u *url.URL, err error) { + return m.client.PresignedGetObject(ctx, m.bucket, objectName, expires, nil) +} + +// MoveObject implements ObjectStorage +func (m *storageMinio) MoveObject(ctx context.Context, srcObject, dstObject string) (err error) { + dst := minio.CopyDestOptions{Bucket: m.bucket, Object: dstObject} + src := minio.CopySrcOptions{Bucket: m.bucket, Object: srcObject} + if _, err := m.client.CopyObject(ctx, dst, src); err != nil { + return fmt.Errorf("move object failed while copy: %v", err) + } + if err := m.client.RemoveObject(ctx, m.bucket, srcObject, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("move object failed while remove: %v", err) + } + return nil +} + +// ObjectExists implements ObjectStorage +func (m *storageMinio) ObjectExists(ctx context.Context, objectName string) (bool, error) { + _, err := m.client.StatObject(ctx, m.bucket, objectName, minio.StatObjectOptions{}) + if err != nil { + er := minio.ToErrorResponse(err) + switch er.Code { + case "NoSuchKey", "NoSuchBucket": + return false, nil + default: + return false, err + } + } + return true, nil +}