captain-neo/octopus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(api): test doc upload

Browse Source
This commit is contained in:
neo-f 2023-03-23 22:28:17 +08:00
parent 6851fe95a0
commit 2d3cf0857a
7 changed files with 45 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/config.yaml
View File

@ -3,7 +3,7 @@ http_port: 8080
prometheus_port: 18090 prometheus_port: 18090
host: http://localhost:8080 host: http://localhost:8080
databases: databases:
storage: minio://admin:uh8Cz62LKFDe9tBHg2PVyDVX7XKqE584xP@10.16.129.140:31141/pangu2-demo storage: minio://admin:uh8Cz62LKFDe9tBHg2PVyDVX7XKqE584xP@10.16.129.140:31141?bucket=pangu2-demo
postgresql: postgres://postgres:TvpHKxDhXzMsVkXWdzwiwUw9KmaLuGdRJo@10.16.129.140:31258/octopus postgresql: postgres://postgres:TvpHKxDhXzMsVkXWdzwiwUw9KmaLuGdRJo@10.16.129.140:31258/octopus
qdrant: 10.16.129.104:32352 qdrant: 10.16.129.104:32352
casdoor: casdoor:

5
internal/dal/model/docs.go
View File

@ -34,10 +34,11 @@ func (f *DocFolder) ParentID() string {
return parts[len(parts)-1] return parts[len(parts)-1]
} }
func (df *DocFolder) ToSchema() *schema.DocFolder { func (df DocFolder) ToSchema() *schema.DocFolder {
return &schema.DocFolder{ return &schema.DocFolder{
ID: df.ID, ID: df.ID,
Name: df.Name, Name: df.Name,
IsDefault: df.IsDefault,
CreatedAt: df.CreatedAt, CreatedAt: df.CreatedAt,
UpdatedAt: df.UpdatedAt, UpdatedAt: df.UpdatedAt,
} }
@ -89,7 +90,7 @@ func (d *Doc) ToSchema(ctx context.Context) *schema.Doc {
return &schema.Doc{ return &schema.Doc{
ID: d.ID, ID: d.ID,
Folder: d.Folder.ToSchema(), Folder: d.Folder.ToSchema(),
PresignedURL: url, PresignedURL: url.String(),
UploadedAt: d.UploadedAt, UploadedAt: d.UploadedAt,
CreatedAt: d.CreatedAt, CreatedAt: d.CreatedAt,
UpdatedAt: d.UpdatedAt, UpdatedAt: d.UpdatedAt,

6
internal/router/doc.go
View File

@ -34,13 +34,13 @@ func RegisterDocRouter(app *soda.Soda) {
AddJWTSecurity(JWTRequired). AddJWTSecurity(JWTRequired).
SetParameters(schema.DocID{}). SetParameters(schema.DocID{}).
AddJSONResponse(200, nil).OK() AddJSONResponse(200, nil).OK()
app.Post("/docs/batch/delete", DeleteDoc). app.Post("/docs/batch/delete", DeleteDocBatch).
AddTags("文档管理"). AddTags("文档管理").
SetSummary("批量-文件删除"). SetSummary("批量-文件删除").
AddJWTSecurity(JWTRequired). AddJWTSecurity(JWTRequired).
SetJSONRequestBody(schema.DocsBatchDelete{}). SetJSONRequestBody(schema.DocsBatchDelete{}).
AddJSONResponse(200, schema.DocBatchResults{}).OK() AddJSONResponse(200, schema.DocBatchResults{}).OK()
app.Post("/docs/batch/update", UpdateDoc). app.Post("/docs/batch/update", UpdateDocBatch).
AddTags("文档管理"). AddTags("文档管理").
SetSummary("批量-文件更新"). SetSummary("批量-文件更新").
AddJWTSecurity(JWTRequired). AddJWTSecurity(JWTRequired).
@ -132,7 +132,7 @@ func CreateUploadURL(c *fiber.Ctx) error {
} }
return c.JSON(schema.UploadURL{ return c.JSON(schema.UploadURL{
URL: *u, URL: u.String(),
ObjectName: objectName, ObjectName: objectName,
}) })
} }

18
internal/schema/docs.go
View File

@ -1,15 +1,15 @@
package schema package schema
import ( import (
"net/url"
"time" "time"
) )
type DocFolder struct { type DocFolder struct {
ID string `json:"id" oai:"description=文件夹ID"` ID string `json:"id" oai:"description=文件夹ID"`
Name string `json:"name" oai:"description=文件夹名称"` Name string `json:"name" oai:"description=文件夹名称"`
CreatedAt time.Time `json:"created_at" oai:"description=创建时间"` IsDefault bool `json:"is_default,omitempty" oai:"description=是否默认分组"`
UpdatedAt time.Time `json:"updated_at" oai:"description=更新时间"` CreatedAt time.Time `json:"created_at" oai:"description=创建时间"`
UpdatedAt time.Time `json:"updated_at" oai:"description=更新时间"`
} }
type DocFolderID struct { type DocFolderID struct {
@ -38,7 +38,7 @@ type DocFolderWithChildren struct {
type Doc struct { type Doc struct {
ID string `json:"id" oai:"description=文档ID"` ID string `json:"id" oai:"description=文档ID"`
Folder *DocFolder `json:"folder" oai:"description=归属文件夹信息"` Folder *DocFolder `json:"folder" oai:"description=归属文件夹信息"`
PresignedURL *url.URL `json:"presigned_url" oai:"description=文档预签名下载URL(临时下载URL)"` PresignedURL string `json:"presigned_url" oai:"description=文档预签名下载URL(临时下载URL)"`
UploadedAt time.Time `json:"uploaded_at" oai:"description=上传时间"` UploadedAt time.Time `json:"uploaded_at" oai:"description=上传时间"`
CreatedAt time.Time `json:"created_at" oai:"description=创建时间"` CreatedAt time.Time `json:"created_at" oai:"description=创建时间"`
UpdatedAt time.Time `json:"updated_at" oai:"description=更新时间"` UpdatedAt time.Time `json:"updated_at" oai:"description=更新时间"`
@ -76,7 +76,7 @@ type DocsBatchDelete struct {
type DocsBatchUpdate struct { type DocsBatchUpdate struct {
IDs []string `json:"ids" validate:"required" oai:"description=批量选择文件ID列表"` IDs []string `json:"ids" validate:"required" oai:"description=批量选择文件ID列表"`
FolderID string `json:"folder_id" validate:"required" oai:"description=更新归属文件夹ID"` FolderID *string `json:"folder_id" oai:"description=更新归属文件夹ID"`
} }
type DocBatchResult struct { type DocBatchResult struct {
@ -92,6 +92,6 @@ type CreateUploadURL struct {
} }
type UploadURL struct { type UploadURL struct {
URL url.URL `json:"url" oai:"description=生成的预签名上传URL"` URL string `json:"url" oai:"description=生成的预签名上传URL"`
ObjectName string `json:"object_name" oai:"description=上传文件名"` ObjectName string `json:"object_name" oai:"description=上传文件名"`
} }

36
internal/service/doc.go
View File

@ -9,7 +9,6 @@ import (
"octopus/internal/dal/model" "octopus/internal/dal/model"
"octopus/internal/dal/query" "octopus/internal/dal/query"
"octopus/internal/schema" "octopus/internal/schema"
"strings"
"time" "time"
"github.com/casdoor/casdoor-go-sdk/casdoorsdk" "github.com/casdoor/casdoor-go-sdk/casdoorsdk"
@ -19,7 +18,9 @@ import (
) )
func ListDocs(ctx context.Context, auth *casdoorsdk.User, param *schema.ListDocQuery) ([]*model.Doc, int64, error) { func ListDocs(ctx context.Context, auth *casdoorsdk.User, param *schema.ListDocQuery) ([]*model.Doc, int64, error) {
base := query.Doc.Where(query.Doc.OrgID.Eq(auth.Owner)) base := query.Doc.
Where(query.Doc.OrgID.Eq(auth.Owner)).
Preload(query.Doc.Folder)
if param.FolderIDs != nil { if param.FolderIDs != nil {
base = base.Where(query.Doc.FolderID.In(*param.FolderIDs...)) base = base.Where(query.Doc.FolderID.In(*param.FolderIDs...))
} }
@ -47,8 +48,8 @@ func CreateDoc(ctx context.Context, auth *casdoorsdk.User, param *schema.CreateD
} }
// 将对象从临时地址移动到新地址 // 将对象从临时地址移动到新地址
newPath := fmt.Sprintf("%s/%s/%s", auth.Owner, folder.ParentPath, param.Name) newObjectName := fmt.Sprintf("%s/%s/%s", auth.Owner, time.Now().UTC().Format(time.DateOnly), param.Name)
if err := storage.MoveObject(ctx, param.ObjectName, newPath); err != nil { if err := storage.MoveObject(ctx, param.ObjectName, newObjectName); err != nil {
return nil, fmt.Errorf("failed to move object: %w", err) return nil, fmt.Errorf("failed to move object: %w", err)
} }
@ -58,9 +59,10 @@ func CreateDoc(ctx context.Context, auth *casdoorsdk.User, param *schema.CreateD
IsDeletable: true, IsDeletable: true,
IsEditable: true, IsEditable: true,
FolderID: param.FolderID, FolderID: param.FolderID,
ObjectName: newPath, ObjectName: newObjectName,
UploadedAt: stat.LastModified, UploadedAt: stat.LastModified,
CreatedBy: auth.Id, CreatedBy: auth.Id,
Folder: folder,
} }
if err := query.Doc.Create(&doc); err != nil { if err := query.Doc.Create(&doc); err != nil {
return nil, fmt.Errorf("failed to create doc: %w", err) return nil, fmt.Errorf("failed to create doc: %w", err)
@ -69,7 +71,13 @@ func CreateDoc(ctx context.Context, auth *casdoorsdk.User, param *schema.CreateD
} }
func GetDoc(ctx context.Context, auth *casdoorsdk.User, id string) (*model.Doc, error) { func GetDoc(ctx context.Context, auth *casdoorsdk.User, id string) (*model.Doc, error) {
doc, err := query.Doc.Where(query.Doc.OrgID.Eq(auth.Owner), query.Doc.ID.Eq(id)).Take() doc, err := query.Doc.
Where(
query.Doc.OrgID.Eq(auth.Owner),
query.Doc.ID.Eq(id),
).
Preload(query.Doc.Folder).
Take()
if errors.Is(err, gorm.ErrRecordNotFound) { if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, fmt.Errorf("specified doc not exists") return nil, fmt.Errorf("specified doc not exists")
} }
@ -80,8 +88,7 @@ func GetDoc(ctx context.Context, auth *casdoorsdk.User, id string) (*model.Doc,
} }
func UpdateDoc(ctx context.Context, auth *casdoorsdk.User, id string, body *schema.UpdateDoc) (*model.Doc, error) { func UpdateDoc(ctx context.Context, auth *casdoorsdk.User, id string, body *schema.UpdateDoc) (*model.Doc, error) {
doc, err := GetDoc(ctx, auth, id) if _, err := GetDoc(ctx, auth, id); err != nil {
if err != nil {
return nil, fmt.Errorf("failed to get doc: %w", err) return nil, fmt.Errorf("failed to get doc: %w", err)
} }
@ -94,15 +101,6 @@ func UpdateDoc(ctx context.Context, auth *casdoorsdk.User, id string, body *sche
if _, err := GetDocFolder(ctx, auth, *body.FolderID); err != nil { if _, err := GetDocFolder(ctx, auth, *body.FolderID); err != nil {
return nil, fmt.Errorf("failed to get doc folder: %w", err) return nil, fmt.Errorf("failed to get doc folder: %w", err)
} }
// 将对象从临时地址移动到新地址
parts := strings.Split(doc.ObjectName, "/")
objname := parts[len(parts)-1]
newObjectName := fmt.Sprintf("%s/%s/%s", auth.Owner, time.Now().UTC().Format(time.DateOnly), objname)
if err := dal.GetStorage().MoveObject(ctx, doc.ObjectName, newObjectName); err != nil {
return nil, fmt.Errorf("failed to move object: %w", err)
}
updates = append(updates, query.Doc.ObjectName.Value(newObjectName))
updates = append(updates, query.Doc.FolderID.Value(*body.FolderID)) updates = append(updates, query.Doc.FolderID.Value(*body.FolderID))
} }
@ -145,7 +143,7 @@ func UpdateDocBatch(ctx context.Context, auth *casdoorsdk.User, param *schema.Do
results := make(schema.DocBatchResults, 0, len(param.IDs)) results := make(schema.DocBatchResults, 0, len(param.IDs))
// NOTE: 后面应该需要优化一下 // NOTE: 后面应该需要优化一下
for _, docID := range param.IDs { for _, docID := range param.IDs {
_, err := UpdateDoc(ctx, auth, docID, &schema.UpdateDoc{FolderID: &param.FolderID}) _, err := UpdateDoc(ctx, auth, docID, &schema.UpdateDoc{FolderID: param.FolderID})
if err != nil { if err != nil {
results = append(results, schema.DocBatchResult{ID: docID, Success: false, Error: err.Error()}) results = append(results, schema.DocBatchResult{ID: docID, Success: false, Error: err.Error()})
} else { } else {
@ -156,7 +154,7 @@ func UpdateDocBatch(ctx context.Context, auth *casdoorsdk.User, param *schema.Do
} }
func CreateUploadURL(ctx context.Context, auth *casdoorsdk.User, param *schema.CreateUploadURL) (u *url.URL, objectName string, err error) { func CreateUploadURL(ctx context.Context, auth *casdoorsdk.User, param *schema.CreateUploadURL) (u *url.URL, objectName string, err error) {
tmpObjectName := fmt.Sprintf("/tmp/%s/%s-%s", auth.Owner, param.FileName, xid.New()) tmpObjectName := fmt.Sprintf("/tmp/%s/%s-%s", auth.Owner, xid.New(), param.FileName)
u, err = dal.GetStorage().PresignedPutObject(ctx, tmpObjectName, time.Hour) u, err = dal.GetStorage().PresignedPutObject(ctx, tmpObjectName, time.Hour)
if err != nil { if err != nil {
return nil, "", fmt.Errorf("failed to create presigned url: %w", err) return nil, "", fmt.Errorf("failed to create presigned url: %w", err)

9
internal/service/doc_folder.go
View File

@ -8,6 +8,7 @@ import (
"octopus/internal/dal/model" "octopus/internal/dal/model"
"octopus/internal/dal/query" "octopus/internal/dal/query"
"octopus/internal/schema" "octopus/internal/schema"
"octopus/pkg/utils"
"github.com/casdoor/casdoor-go-sdk/casdoorsdk" "github.com/casdoor/casdoor-go-sdk/casdoorsdk"
"gorm.io/gen/field" "gorm.io/gen/field"
@ -38,7 +39,7 @@ func GetDocFolderTree(ctx context.Context, auth *casdoorsdk.User, param *schema.
} }
// 如果没有的话，就创建一个默认的分组 // 如果没有的话，就创建一个默认的分组
if len(folders) == 0 { if utils.Unptr(param.ParentID) == "" && len(folders) == 0 {
folder := model.DocFolder{ folder := model.DocFolder{
Base: model.Base{OrgID: auth.Owner}, Base: model.Base{OrgID: auth.Owner},
Name: "默认分组", Name: "默认分组",
@ -132,9 +133,9 @@ func DeleteDocFolder(ctx context.Context, auth *casdoorsdk.User, id string) erro
if err != nil { if err != nil {
return fmt.Errorf("failed to get doc folder: %w", err) return fmt.Errorf("failed to get doc folder: %w", err)
} }
if folder.IsDefault { // if folder.IsDefault {
return fmt.Errorf("cannot delete the default folder") // return fmt.Errorf("cannot delete the default folder")
} // }
subFolders, err := GetDocFolderChildren(ctx, auth, id) subFolders, err := GetDocFolderChildren(ctx, auth, id)
if err != nil { if err != nil {
return fmt.Errorf("failed to get doc folder children: %w", err) return fmt.Errorf("failed to get doc folder children: %w", err)

16
pkg/storage/storage_minio.go
View File

@ -86,16 +86,14 @@ func (m *StorageMinio) DeleteObject(ctx context.Context, objectName string) (err
// MoveObject implements ObjectStorage // MoveObject implements ObjectStorage
func (m *StorageMinio) DeleteObjects(ctx context.Context, objectNames []string) []minio.RemoveObjectError { func (m *StorageMinio) DeleteObjects(ctx context.Context, objectNames []string) []minio.RemoveObjectError {
objs := make(chan minio.ObjectInfo, len(objectNames)) // FIXME: why does DeleteObjects not working??
for _, objectName := range objectNames {
objs <- minio.ObjectInfo{Key: objectName}
}
errCh := m.client.RemoveObjects(ctx, m.bucket, objs, minio.RemoveObjectsOptions{})
var errors []minio.RemoveObjectError var errors []minio.RemoveObjectError
for err := range errCh { for _, objectName := range objectNames {
if err.Err != nil { if err := m.DeleteObject(ctx, objectName); err != nil {
errors = append(errors, err) errors = append(errors, minio.RemoveObjectError{
ObjectName: objectName,
Err: err,
})
} }
} }
return errors return errors