Init V4 community edition (#2265)
* Init V4 community edition * Init V4 community edition
This commit is contained in:
AaronLiu
588
pkg/filemanager/driver/cos/cos.go
Normal file
588
pkg/filemanager/driver/cos/cos.go
Normal file
@@ -0,0 +1,588 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v4/ent"
|
||||
"github.com/cloudreve/Cloudreve/v4/inventory/types"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/cluster/routes"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/mime"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/util"
|
||||
"github.com/google/go-querystring/query"
|
||||
"github.com/samber/lo"
|
||||
cossdk "github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
|
||||
// UploadPolicy 腾讯云COS上传策略
|
||||
type UploadPolicy struct {
|
||||
Expiration string `json:"expiration"`
|
||||
Conditions []interface{} `json:"conditions"`
|
||||
}
|
||||
|
||||
// MetaData 文件元信息
|
||||
type MetaData struct {
|
||||
Size uint64
|
||||
CallbackKey string
|
||||
CallbackURL string
|
||||
}
|
||||
|
||||
type urlOption struct {
|
||||
Speed int64 `url:"x-cos-traffic-limit,omitempty"`
|
||||
ContentDescription string `url:"response-content-disposition,omitempty"`
|
||||
Exif *string `url:"exif,omitempty"`
|
||||
CiProcess string `url:"ci-process,omitempty"`
|
||||
}
|
||||
|
||||
type (
|
||||
CosParts struct {
|
||||
ETag string
|
||||
PartNumber int
|
||||
}
|
||||
)
|
||||
|
||||
// Driver 腾讯云COS适配器模板
|
||||
type Driver struct {
|
||||
policy *ent.StoragePolicy
|
||||
client *cossdk.Client
|
||||
settings setting.Provider
|
||||
config conf.ConfigProvider
|
||||
httpClient request.Client
|
||||
l logging.Logger
|
||||
mime mime.MimeDetector
|
||||
|
||||
chunkSize int64
|
||||
}
|
||||
|
||||
const (
|
||||
// MultiPartUploadThreshold 服务端使用分片上传的阈值
|
||||
MultiPartUploadThreshold int64 = 5 * (1 << 30) // 5GB
|
||||
|
||||
maxDeleteBatch = 1000
|
||||
chunkRetrySleep = time.Duration(5) * time.Second
|
||||
overwriteOptionHeader = "x-cos-forbid-overwrite"
|
||||
partNumberParam = "partNumber"
|
||||
uploadIdParam = "uploadId"
|
||||
contentTypeHeader = "Content-Type"
|
||||
contentLengthHeader = "Content-Length"
|
||||
)
|
||||
|
||||
var (
|
||||
features = &boolset.BooleanSet{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cossdk.SetNeedSignHeaders("host", false)
|
||||
cossdk.SetNeedSignHeaders("origin", false)
|
||||
boolset.Sets(map[driver.HandlerCapability]bool{
|
||||
driver.HandlerCapabilityUploadSentinelRequired: true,
|
||||
}, features)
|
||||
}
|
||||
|
||||
func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provider,
|
||||
config conf.ConfigProvider, l logging.Logger, mime mime.MimeDetector) (*Driver, error) {
|
||||
chunkSize := policy.Settings.ChunkSize
|
||||
if policy.Settings.ChunkSize == 0 {
|
||||
chunkSize = 25 << 20 // 25 MB
|
||||
}
|
||||
|
||||
driver := &Driver{
|
||||
policy: policy,
|
||||
settings: settings,
|
||||
chunkSize: chunkSize,
|
||||
config: config,
|
||||
l: l,
|
||||
mime: mime,
|
||||
httpClient: request.NewClient(config, request.WithLogger(l)),
|
||||
}
|
||||
|
||||
u, err := url.Parse(policy.Server)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse COS bucket server url: %w", err)
|
||||
}
|
||||
driver.client = cossdk.NewClient(&cossdk.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cossdk.AuthorizationTransport{
|
||||
SecretID: policy.AccessKey,
|
||||
SecretKey: policy.SecretKey,
|
||||
},
|
||||
})
|
||||
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
//
|
||||
//// List 列出COS文件
|
||||
//func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// // 初始化列目录参数
|
||||
// opt := &cossdk.BucketGetOptions{
|
||||
// Prefix: strings.TrimPrefix(base, "/"),
|
||||
// EncodingType: "",
|
||||
// MaxKeys: 1000,
|
||||
// }
|
||||
// // 是否为递归列出
|
||||
// if !recursive {
|
||||
// opt.Delimiter = "/"
|
||||
// }
|
||||
// // 手动补齐结尾的slash
|
||||
// if opt.Prefix != "" {
|
||||
// opt.Prefix += "/"
|
||||
// }
|
||||
//
|
||||
// var (
|
||||
// marker string
|
||||
// objects []cossdk.Object
|
||||
// commons []string
|
||||
// )
|
||||
//
|
||||
// for {
|
||||
// res, _, err := handler.client.Bucket.Get(ctx, opt)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// objects = append(objects, res.Contents...)
|
||||
// commons = append(commons, res.CommonPrefixes...)
|
||||
// // 如果本次未列取完，则继续使用marker获取结果
|
||||
// marker = res.NextMarker
|
||||
// // marker 为空时结果列取完毕，跳出
|
||||
// if marker == "" {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // 处理列取结果
|
||||
// res := make([]response.Object, 0, len(objects)+len(commons))
|
||||
// // 处理目录
|
||||
// for _, object := range commons {
|
||||
// rel, err := filepath.Rel(opt.Prefix, object)
|
||||
// if err != nil {
|
||||
// continue
|
||||
// }
|
||||
// res = append(res, response.Object{
|
||||
// Name: path.Base(object),
|
||||
// RelativePath: filepath.ToSlash(rel),
|
||||
// Size: 0,
|
||||
// IsDir: true,
|
||||
// LastModify: time.Now(),
|
||||
// })
|
||||
// }
|
||||
// // 处理文件
|
||||
// for _, object := range objects {
|
||||
// rel, err := filepath.Rel(opt.Prefix, object.Key)
|
||||
// if err != nil {
|
||||
// continue
|
||||
// }
|
||||
// res = append(res, response.Object{
|
||||
// Name: path.Base(object.Key),
|
||||
// Source: object.Key,
|
||||
// RelativePath: filepath.ToSlash(rel),
|
||||
// Size: uint64(object.Size),
|
||||
// IsDir: false,
|
||||
// LastModify: time.Now(),
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// return res, nil
|
||||
//
|
||||
//}
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler Driver) CORS() error {
|
||||
_, err := handler.client.Bucket.PutCORS(context.Background(), &cossdk.BucketPutCORSOptions{
|
||||
Rules: []cossdk.BucketCORSRule{{
|
||||
AllowedMethods: []string{
|
||||
"GET",
|
||||
"POST",
|
||||
"PUT",
|
||||
"DELETE",
|
||||
"HEAD",
|
||||
},
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
MaxAgeSeconds: 3600,
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
}},
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error {
|
||||
defer file.Close()
|
||||
|
||||
mimeType := file.Props.MimeType
|
||||
if mimeType == "" {
|
||||
handler.mime.TypeByName(file.Props.Uri.Name())
|
||||
}
|
||||
|
||||
// 是否允许覆盖
|
||||
overwrite := file.Mode&fs.ModeOverwrite == fs.ModeOverwrite
|
||||
opt := &cossdk.ObjectPutHeaderOptions{
|
||||
ContentType: mimeType,
|
||||
XOptionHeader: &http.Header{
|
||||
overwriteOptionHeader: []string{fmt.Sprintf("%t", overwrite)},
|
||||
},
|
||||
}
|
||||
|
||||
// 小文件直接上传
|
||||
if file.Props.Size < MultiPartUploadThreshold {
|
||||
_, err := handler.client.Object.Put(ctx, file.Props.SavePath, file, &cossdk.ObjectPutOptions{
|
||||
ObjectPutHeaderOptions: opt,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
imur, _, err := handler.client.Object.InitiateMultipartUpload(ctx, file.Props.SavePath, &cossdk.InitiateMultipartUploadOptions{
|
||||
ObjectPutHeaderOptions: opt,
|
||||
})
|
||||
|
||||
chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{
|
||||
Max: handler.settings.ChunkRetryLimit(ctx),
|
||||
Sleep: chunkRetrySleep,
|
||||
}, handler.settings.UseChunkBuffer(ctx), handler.l, handler.settings.TempPath(ctx))
|
||||
|
||||
parts := make([]CosParts, 0, chunks.Num())
|
||||
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
|
||||
res, err := handler.client.Object.UploadPart(ctx, file.Props.SavePath, imur.UploadID, current.Index()+1, content, &cossdk.ObjectUploadPartOptions{
|
||||
ContentLength: current.Length(),
|
||||
})
|
||||
if err == nil {
|
||||
parts = append(parts, CosParts{
|
||||
ETag: res.Header.Get("ETag"),
|
||||
PartNumber: current.Index() + 1,
|
||||
})
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for chunks.Next() {
|
||||
if err := chunks.Process(uploadFunc); err != nil {
|
||||
handler.cancelUpload(file.Props.SavePath, imur.UploadID)
|
||||
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
|
||||
}
|
||||
}
|
||||
|
||||
_, _, err = handler.client.Object.CompleteMultipartUpload(ctx, file.Props.SavePath, imur.UploadID, &cossdk.CompleteMultipartUploadOptions{
|
||||
Parts: lo.Map(parts, func(v CosParts, i int) cossdk.Object {
|
||||
return cossdk.Object{
|
||||
ETag: v.ETag,
|
||||
PartNumber: v.PartNumber,
|
||||
}
|
||||
}),
|
||||
XOptionHeader: &http.Header{
|
||||
overwriteOptionHeader: []string{fmt.Sprintf("%t", overwrite)},
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
handler.cancelUpload(file.Props.SavePath, imur.UploadID)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件，
|
||||
// 返回未删除的文件，及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files ...string) ([]string, error) {
|
||||
groups := lo.Chunk(files, maxDeleteBatch)
|
||||
failed := make([]string, 0)
|
||||
var lastError error
|
||||
for index, group := range groups {
|
||||
handler.l.Debug("Process delete group #%d: %v", index, group)
|
||||
res, _, err := handler.client.Object.DeleteMulti(ctx,
|
||||
&cossdk.ObjectDeleteMultiOptions{
|
||||
Objects: lo.Map(group, func(item string, index int) cossdk.Object {
|
||||
return cossdk.Object{Key: item}
|
||||
}),
|
||||
Quiet: true,
|
||||
})
|
||||
if err != nil {
|
||||
lastError = err
|
||||
failed = append(failed, group...)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, v := range res.Errors {
|
||||
handler.l.Debug("Failed to delete file: %s, Code:%s, Message:%s", v.Key, v.Code, v.Key)
|
||||
failed = append(failed, v.Key)
|
||||
}
|
||||
}
|
||||
|
||||
if len(failed) > 0 && lastError == nil {
|
||||
lastError = fmt.Errorf("failed to delete files: %v", failed)
|
||||
}
|
||||
|
||||
return failed, lastError
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, expire *time.Time, ext string, e fs.Entity) (string, error) {
|
||||
w, h := handler.settings.ThumbSize(ctx)
|
||||
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d", w, h)
|
||||
|
||||
source, err := handler.signSourceURL(
|
||||
ctx,
|
||||
e.Source(),
|
||||
expire,
|
||||
&urlOption{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
thumbURL, _ := url.Parse(source)
|
||||
thumbQuery := thumbURL.Query()
|
||||
thumbQuery.Add(thumbParam, "")
|
||||
thumbURL.RawQuery = thumbQuery.Encode()
|
||||
|
||||
return thumbURL.String(), nil
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(ctx context.Context, e fs.Entity, args *driver.GetSourceArgs) (string, error) {
|
||||
// 添加各项设置
|
||||
options := urlOption{}
|
||||
if args.Speed > 0 {
|
||||
if args.Speed < 819200 {
|
||||
args.Speed = 819200
|
||||
}
|
||||
if args.Speed > 838860800 {
|
||||
args.Speed = 838860800
|
||||
}
|
||||
options.Speed = args.Speed
|
||||
}
|
||||
if args.IsDownload {
|
||||
encodedFilename := url.PathEscape(args.DisplayName)
|
||||
options.ContentDescription = fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`,
|
||||
encodedFilename, encodedFilename)
|
||||
}
|
||||
|
||||
return handler.signSourceURL(ctx, e.Source(), args.Expire, &options)
|
||||
}
|
||||
|
||||
func (handler Driver) signSourceURL(ctx context.Context, path string, expire *time.Time, options *urlOption) (string, error) {
|
||||
// 公有空间不需要签名
|
||||
if !handler.policy.IsPrivate || (handler.policy.Settings.SourceAuth && handler.policy.Settings.CustomProxy) {
|
||||
file, err := url.Parse(handler.policy.Server)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
file.Path = path
|
||||
|
||||
// 非签名URL不支持设置响应header
|
||||
options.ContentDescription = ""
|
||||
|
||||
optionQuery, err := query.Values(*options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
file.RawQuery = optionQuery.Encode()
|
||||
|
||||
return file.String(), nil
|
||||
}
|
||||
|
||||
ttl := time.Duration(0)
|
||||
if expire != nil {
|
||||
ttl = time.Until(*expire)
|
||||
} else {
|
||||
// 20 years for permanent link
|
||||
ttl = time.Duration(24) * time.Hour * 365 * 20
|
||||
}
|
||||
|
||||
presignedURL, err := handler.client.Object.GetPresignedURL(ctx, http.MethodGet, path,
|
||||
handler.policy.AccessKey, handler.policy.SecretKey, ttl, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, uploadSession *fs.UploadSession, file *fs.UploadRequest) (*fs.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
siteURL := handler.settings.SiteURL(setting.UseFirstSiteUrl(ctx))
|
||||
// 在从机端创建上传会话
|
||||
uploadSession.ChunkSize = handler.chunkSize
|
||||
uploadSession.Callback = routes.MasterSlaveCallbackUrl(siteURL, types.PolicyTypeCos, uploadSession.Props.UploadSessionID, uploadSession.CallbackSecret).String()
|
||||
|
||||
mimeType := file.Props.MimeType
|
||||
if mimeType == "" {
|
||||
handler.mime.TypeByName(file.Props.Uri.Name())
|
||||
}
|
||||
|
||||
// 初始化分片上传
|
||||
opt := &cossdk.ObjectPutHeaderOptions{
|
||||
ContentType: mimeType,
|
||||
XOptionHeader: &http.Header{
|
||||
overwriteOptionHeader: []string{"true"},
|
||||
},
|
||||
}
|
||||
|
||||
imur, _, err := handler.client.Object.InitiateMultipartUpload(ctx, file.Props.SavePath, &cossdk.InitiateMultipartUploadOptions{
|
||||
ObjectPutHeaderOptions: opt,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
|
||||
}
|
||||
uploadSession.UploadID = imur.UploadID
|
||||
|
||||
// 为每个分片签名上传 URL
|
||||
chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{}, false, handler.l, "")
|
||||
urls := make([]string, chunks.Num())
|
||||
ttl := time.Until(uploadSession.Props.ExpireAt)
|
||||
for chunks.Next() {
|
||||
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
|
||||
signedURL, err := handler.client.Object.GetPresignedURL(
|
||||
ctx,
|
||||
http.MethodPut,
|
||||
file.Props.SavePath,
|
||||
handler.policy.AccessKey,
|
||||
handler.policy.SecretKey,
|
||||
ttl,
|
||||
&cossdk.PresignedURLOptions{
|
||||
Query: &url.Values{
|
||||
partNumberParam: []string{fmt.Sprintf("%d", c.Index()+1)},
|
||||
uploadIdParam: []string{imur.UploadID},
|
||||
},
|
||||
Header: &http.Header{
|
||||
contentTypeHeader: []string{"application/octet-stream"},
|
||||
contentLengthHeader: []string{fmt.Sprintf("%d", c.Length())},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
urls[c.Index()] = signedURL.String()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 签名完成分片上传的URL
|
||||
completeURL, err := handler.client.Object.GetPresignedURL(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
file.Props.SavePath,
|
||||
handler.policy.AccessKey,
|
||||
handler.policy.SecretKey,
|
||||
time.Until(uploadSession.Props.ExpireAt),
|
||||
&cossdk.PresignedURLOptions{
|
||||
Query: &url.Values{
|
||||
uploadIdParam: []string{imur.UploadID},
|
||||
},
|
||||
Header: &http.Header{
|
||||
overwriteOptionHeader: []string{"true"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &fs.UploadCredential{
|
||||
UploadID: imur.UploadID,
|
||||
UploadURLs: urls,
|
||||
CompleteURL: completeURL.String(),
|
||||
SessionID: uploadSession.Props.UploadSessionID,
|
||||
ChunkSize: handler.chunkSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *fs.UploadSession) error {
|
||||
_, err := handler.client.Object.AbortMultipartUpload(ctx, uploadSession.Props.SavePath, uploadSession.UploadID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (handler *Driver) CompleteUpload(ctx context.Context, session *fs.UploadSession) error {
|
||||
if session.SentinelTaskID == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Make sure uploaded file size is correct
|
||||
res, err := handler.client.Object.Head(ctx, session.Props.SavePath, &cossdk.ObjectHeadOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get uploaded file size: %w", err)
|
||||
}
|
||||
|
||||
if res.ContentLength != session.Props.Size {
|
||||
return serializer.NewError(
|
||||
serializer.CodeMetaMismatch,
|
||||
fmt.Sprintf("File size not match, expected: %d, actual: %d", session.Props.Size, res.ContentLength),
|
||||
nil,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *Driver) Capabilities() *driver.Capabilities {
|
||||
mediaMetaExts := handler.policy.Settings.MediaMetaExts
|
||||
if !handler.policy.Settings.NativeMediaProcessing {
|
||||
mediaMetaExts = nil
|
||||
}
|
||||
return &driver.Capabilities{
|
||||
StaticFeatures: features,
|
||||
MediaMetaSupportedExts: mediaMetaExts,
|
||||
MediaMetaProxy: handler.policy.Settings.MediaMetaGeneratorProxy,
|
||||
ThumbSupportedExts: handler.policy.Settings.ThumbExts,
|
||||
ThumbProxy: handler.policy.Settings.ThumbGeneratorProxy,
|
||||
ThumbMaxSize: handler.policy.Settings.ThumbMaxSize,
|
||||
ThumbSupportAllExts: handler.policy.Settings.ThumbSupportAllExts,
|
||||
}
|
||||
}
|
||||
|
||||
// Meta 获取文件信息
|
||||
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
|
||||
res, err := handler.client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MetaData{
|
||||
Size: uint64(res.ContentLength),
|
||||
CallbackKey: res.Header.Get("x-cos-meta-key"),
|
||||
CallbackURL: res.Header.Get("x-cos-meta-callback"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler *Driver) MediaMeta(ctx context.Context, path, ext string) ([]driver.MediaMeta, error) {
|
||||
if util.ContainsString(supportedImageExt, ext) {
|
||||
return handler.extractImageMeta(ctx, path)
|
||||
}
|
||||
|
||||
return handler.extractStreamMeta(ctx, path)
|
||||
}
|
||||
|
||||
func (handler *Driver) LocalPath(ctx context.Context, path string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (handler *Driver) cancelUpload(path, uploadId string) {
|
||||
if _, err := handler.client.Object.AbortMultipartUpload(context.Background(), path, uploadId); err != nil {
|
||||
handler.l.Warning("failed to abort multipart upload: %s", err)
|
||||
}
|
||||
}
|
||||
294
pkg/filemanager/driver/cos/media.go
Normal file
294
pkg/filemanager/driver/cos/media.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/mediameta"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"github.com/samber/lo"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
mediaInfoTTL = time.Duration(10) * time.Minute
|
||||
videoInfo = "videoinfo"
|
||||
)
|
||||
|
||||
var (
|
||||
supportedImageExt = []string{"jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff", "heic", "heif"}
|
||||
)
|
||||
|
||||
type (
|
||||
ImageProp struct {
|
||||
Value string `json:"val"`
|
||||
}
|
||||
ImageInfo map[string]ImageProp
|
||||
Error struct {
|
||||
XMLName xml.Name `xml:"Error"`
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
RequestId string `xml:"RequestId"`
|
||||
}
|
||||
Video struct {
|
||||
Index int `xml:"Index"`
|
||||
CodecName string `xml:"CodecName"`
|
||||
CodecLongName string `xml:"CodecLongName"`
|
||||
CodecTimeBase string `xml:"CodecTimeBase"`
|
||||
CodecTagString string `xml:"CodecTagString"`
|
||||
CodecTag string `xml:"CodecTag"`
|
||||
ColorPrimaries string `xml:"ColorPrimaries"`
|
||||
ColorRange string `xml:"ColorRange"`
|
||||
ColorTransfer string `xml:"ColorTransfer"`
|
||||
Profile string `xml:"Profile"`
|
||||
Width int `xml:"Width"`
|
||||
Height int `xml:"Height"`
|
||||
HasBFrame string `xml:"HasBFrame"`
|
||||
RefFrames string `xml:"RefFrames"`
|
||||
Sar string `xml:"Sar"`
|
||||
Dar string `xml:"Dar"`
|
||||
PixFormat string `xml:"PixFormat"`
|
||||
FieldOrder string `xml:"FieldOrder"`
|
||||
Level string `xml:"Level"`
|
||||
Fps string `xml:"Fps"`
|
||||
AvgFps string `xml:"AvgFps"`
|
||||
Timebase string `xml:"Timebase"`
|
||||
StartTime string `xml:"StartTime"`
|
||||
Duration string `xml:"Duration"`
|
||||
Bitrate string `xml:"Bitrate"`
|
||||
NumFrames string `xml:"NumFrames"`
|
||||
Language string `xml:"Language"`
|
||||
}
|
||||
Audio struct {
|
||||
Index int `xml:"Index"`
|
||||
CodecName string `xml:"CodecName"`
|
||||
CodecLongName string `xml:"CodecLongName"`
|
||||
CodecTimeBase string `xml:"CodecTimeBase"`
|
||||
CodecTagString string `xml:"CodecTagString"`
|
||||
CodecTag string `xml:"CodecTag"`
|
||||
SampleFmt string `xml:"SampleFmt"`
|
||||
SampleRate string `xml:"SampleRate"`
|
||||
Channel string `xml:"Channel"`
|
||||
ChannelLayout string `xml:"ChannelLayout"`
|
||||
Timebase string `xml:"Timebase"`
|
||||
StartTime string `xml:"StartTime"`
|
||||
Duration string `xml:"Duration"`
|
||||
Bitrate string `xml:"Bitrate"`
|
||||
Language string `xml:"Language"`
|
||||
}
|
||||
Subtitle struct {
|
||||
Index string `xml:"Index"`
|
||||
Language string `xml:"Language"`
|
||||
}
|
||||
Response struct {
|
||||
XMLName xml.Name `xml:"Response"`
|
||||
MediaInfo struct {
|
||||
Stream struct {
|
||||
Video []Video `xml:"Video"`
|
||||
Audio []Audio `xml:"Audio"`
|
||||
Subtitle []Subtitle `xml:"Subtitle"`
|
||||
} `xml:"Stream"`
|
||||
Format struct {
|
||||
NumStream string `xml:"NumStream"`
|
||||
NumProgram string `xml:"NumProgram"`
|
||||
FormatName string `xml:"FormatName"`
|
||||
FormatLongName string `xml:"FormatLongName"`
|
||||
StartTime string `xml:"StartTime"`
|
||||
Duration string `xml:"Duration"`
|
||||
Bitrate string `xml:"Bitrate"`
|
||||
Size string `xml:"Size"`
|
||||
} `xml:"Format"`
|
||||
} `xml:"MediaInfo"`
|
||||
}
|
||||
)
|
||||
|
||||
func (handler *Driver) extractStreamMeta(ctx context.Context, path string) ([]driver.MediaMeta, error) {
|
||||
resp, err := handler.extractMediaInfo(ctx, path, &urlOption{CiProcess: videoInfo})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var info Response
|
||||
if err := xml.Unmarshal([]byte(resp), &info); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
|
||||
}
|
||||
|
||||
streams := lo.Map(info.MediaInfo.Stream.Video, func(stream Video, index int) mediameta.Stream {
|
||||
return mediameta.Stream{
|
||||
Index: stream.Index,
|
||||
CodecName: stream.CodecName,
|
||||
CodecLongName: stream.CodecLongName,
|
||||
CodecType: "video",
|
||||
Width: stream.Width,
|
||||
Height: stream.Height,
|
||||
Bitrate: stream.Bitrate,
|
||||
}
|
||||
})
|
||||
streams = append(streams, lo.Map(info.MediaInfo.Stream.Audio, func(stream Audio, index int) mediameta.Stream {
|
||||
return mediameta.Stream{
|
||||
Index: stream.Index,
|
||||
CodecName: stream.CodecName,
|
||||
CodecLongName: stream.CodecLongName,
|
||||
CodecType: "audio",
|
||||
Bitrate: stream.Bitrate,
|
||||
}
|
||||
})...)
|
||||
|
||||
metas := make([]driver.MediaMeta, 0)
|
||||
metas = append(metas, mediameta.ProbeMetaTransform(&mediameta.FFProbeMeta{
|
||||
Format: &mediameta.Format{
|
||||
FormatName: info.MediaInfo.Format.FormatName,
|
||||
FormatLongName: info.MediaInfo.Format.FormatLongName,
|
||||
Duration: info.MediaInfo.Format.Duration,
|
||||
Bitrate: info.MediaInfo.Format.Bitrate,
|
||||
},
|
||||
Streams: streams,
|
||||
})...)
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (handler *Driver) extractImageMeta(ctx context.Context, path string) ([]driver.MediaMeta, error) {
|
||||
exif := ""
|
||||
resp, err := handler.extractMediaInfo(ctx, path, &urlOption{
|
||||
Exif: &exif,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var imageInfo ImageInfo
|
||||
if err := json.Unmarshal([]byte(resp), &imageInfo); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
|
||||
}
|
||||
|
||||
metas := make([]driver.MediaMeta, 0)
|
||||
exifMap := lo.MapEntries(imageInfo, func(key string, value ImageProp) (string, string) {
|
||||
return key, value.Value
|
||||
})
|
||||
metas = append(metas, mediameta.ExtractExifMap(exifMap, time.Time{})...)
|
||||
metas = append(metas, parseGpsInfo(imageInfo)...)
|
||||
for i := 0; i < len(metas); i++ {
|
||||
metas[i].Type = driver.MetaTypeExif
|
||||
}
|
||||
|
||||
return metas, nil
|
||||
}
|
||||
|
||||
// extractMediaInfo Sends API calls to COS service to extract media info.
|
||||
func (handler *Driver) extractMediaInfo(ctx context.Context, path string, opt *urlOption) (string, error) {
|
||||
mediaInfoExpire := time.Now().Add(mediaInfoTTL)
|
||||
thumbURL, err := handler.signSourceURL(
|
||||
ctx,
|
||||
path,
|
||||
&mediaInfoExpire,
|
||||
opt,
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to sign media info url: %w", err)
|
||||
}
|
||||
|
||||
resp, err := handler.httpClient.
|
||||
Request(http.MethodGet, thumbURL, nil, request.WithContext(ctx)).
|
||||
CheckHTTPResponse(http.StatusOK).
|
||||
GetResponseIgnoreErr()
|
||||
if err != nil {
|
||||
return "", handleCosError(resp, err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func parseGpsInfo(imageInfo ImageInfo) []driver.MediaMeta {
|
||||
latitude := imageInfo["GPSLatitude"] // 31deg 16.26808'
|
||||
longitude := imageInfo["GPSLongitude"] // 120deg 42.91039'
|
||||
latRef := imageInfo["GPSLatitudeRef"] // North
|
||||
lonRef := imageInfo["GPSLongitudeRef"] // East
|
||||
|
||||
// Make sure all value exist in map
|
||||
if latitude.Value == "" || longitude.Value == "" || latRef.Value == "" || lonRef.Value == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
lat := parseRawGPS(latitude.Value, latRef.Value)
|
||||
lon := parseRawGPS(longitude.Value, lonRef.Value)
|
||||
if !math.IsNaN(lat) && !math.IsNaN(lon) {
|
||||
lat, lng := mediameta.NormalizeGPS(lat, lon)
|
||||
return []driver.MediaMeta{{
|
||||
Key: mediameta.GpsLat,
|
||||
Value: fmt.Sprintf("%f", lat),
|
||||
}, {
|
||||
Key: mediameta.GpsLng,
|
||||
Value: fmt.Sprintf("%f", lng),
|
||||
}}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseRawGPS(gpsStr string, ref string) float64 {
|
||||
elem := strings.Split(gpsStr, " ")
|
||||
if len(elem) < 1 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var (
|
||||
deg float64
|
||||
minutes float64
|
||||
seconds float64
|
||||
)
|
||||
|
||||
deg = getGpsElemValue(elem[0])
|
||||
if len(elem) >= 2 {
|
||||
minutes = getGpsElemValue(elem[1])
|
||||
}
|
||||
if len(elem) >= 3 {
|
||||
seconds = getGpsElemValue(elem[2])
|
||||
}
|
||||
|
||||
decimal := deg + minutes/60.0 + seconds/3600.0
|
||||
|
||||
if ref == "S" || ref == "W" {
|
||||
return -decimal
|
||||
}
|
||||
|
||||
return decimal
|
||||
}
|
||||
|
||||
func getGpsElemValue(elm string) float64 {
|
||||
elements := strings.Split(elm, "/")
|
||||
if len(elements) != 2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
numerator, err := strconv.ParseFloat(elements[0], 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
denominator, err := strconv.ParseFloat(elements[1], 64)
|
||||
if err != nil || denominator == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return numerator / denominator
|
||||
}
|
||||
|
||||
func handleCosError(resp string, originErr error) error {
|
||||
if resp == "" {
|
||||
return originErr
|
||||
}
|
||||
|
||||
var err Error
|
||||
if err := xml.Unmarshal([]byte(resp), &err); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal cos error: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("cos error: %s", err.Message)
|
||||
}
|
||||
118
pkg/filemanager/driver/cos/scf.go
Normal file
118
pkg/filemanager/driver/cos/scf.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package cos
|
||||
|
||||
// TODO: revisit para error
|
||||
const scfFunc = `# -*- coding: utf8 -*-
|
||||
# SCF配置COS触发，向 Cloudreve 发送回调
|
||||
from qcloud_cos_v5 import CosConfig
|
||||
from qcloud_cos_v5 import CosS3Client
|
||||
from qcloud_cos_v5 import CosServiceError
|
||||
from qcloud_cos_v5 import CosClientError
|
||||
import sys
|
||||
import logging
|
||||
import requests
|
||||
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
logging = logging.getLogger()
|
||||
|
||||
|
||||
def main_handler(event, context):
|
||||
logging.info("start main handler")
|
||||
for record in event['Records']:
|
||||
try:
|
||||
if "x-cos-meta-callback" not in record['cos']['cosObject']['meta']:
|
||||
logging.info("Cannot find callback URL, skiped.")
|
||||
return 'Success'
|
||||
callback = record['cos']['cosObject']['meta']['x-cos-meta-callback']
|
||||
key = record['cos']['cosObject']['key']
|
||||
logging.info("Callback URL is " + callback)
|
||||
|
||||
r = requests.get(callback)
|
||||
print(r.text)
|
||||
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print('Error getting object {} callback url. '.format(key))
|
||||
raise e
|
||||
return "Fail"
|
||||
|
||||
return "Success"
|
||||
`
|
||||
|
||||
//
|
||||
//// CreateSCF 创建回调云函数
|
||||
//func CreateSCF(policy *model.Policy, region string) error {
|
||||
// // 初始化客户端
|
||||
// credential := common.NewCredential(
|
||||
// policy.AccessKey,
|
||||
// policy.SecretKey,
|
||||
// )
|
||||
// cpf := profile.NewClientProfile()
|
||||
// client, err := scf.NewClient(credential, region, cpf)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// // 创建回调代码数据
|
||||
// buff := &bytes.Buffer{}
|
||||
// bs64 := base64.NewEncoder(base64.StdEncoding, buff)
|
||||
// zipWriter := zip.NewWriter(bs64)
|
||||
// header := zip.FileHeader{
|
||||
// Name: "callback.py",
|
||||
// Method: zip.Deflate,
|
||||
// }
|
||||
// writer, err := zipWriter.CreateHeader(&header)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _, err = io.Copy(writer, strings.NewReader(scfFunc))
|
||||
// zipWriter.Close()
|
||||
//
|
||||
// // 创建云函数
|
||||
// req := scf.NewCreateFunctionRequest()
|
||||
// funcName := "cloudreve_" + hashid.HashID(policy.ID, hashid.PolicyID) + strconv.FormatInt(time.Now().Unix(), 10)
|
||||
// zipFileBytes, _ := ioutil.ReadAll(buff)
|
||||
// zipFileStr := string(zipFileBytes)
|
||||
// codeSource := "ZipFile"
|
||||
// handler := "callback.main_handler"
|
||||
// desc := "Cloudreve 用回调函数"
|
||||
// timeout := int64(60)
|
||||
// runtime := "Python3.6"
|
||||
// req.FunctionName = &funcName
|
||||
// req.Code = &scf.Code{
|
||||
// ZipFile: &zipFileStr,
|
||||
// }
|
||||
// req.Handler = &handler
|
||||
// req.Description = &desc
|
||||
// req.Timeout = &timeout
|
||||
// req.Runtime = &runtime
|
||||
// req.CodeSource = &codeSource
|
||||
//
|
||||
// _, err = client.CreateFunction(req)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// time.Sleep(time.Duration(5) * time.Second)
|
||||
//
|
||||
// // 创建触发器
|
||||
// server, _ := url.Parse(policy.Server)
|
||||
// triggerType := "cos"
|
||||
// triggerDesc := `{"event":"cos:ObjectCreated:Post","filter":{"Prefix":"","Suffix":""}}`
|
||||
// enable := "OPEN"
|
||||
//
|
||||
// trigger := scf.NewCreateTriggerRequest()
|
||||
// trigger.FunctionName = &funcName
|
||||
// trigger.TriggerName = &server.Host
|
||||
// trigger.Type = &triggerType
|
||||
// trigger.TriggerDesc = &triggerDesc
|
||||
// trigger.Enable = &enable
|
||||
//
|
||||
// _, err = client.CreateTrigger(trigger)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// return nil
|
||||
//}
|
||||
Reference in New Issue
Block a user