Init V4 community edition (#2265)

* Init V4 community edition

* Init V4 community edition
This commit is contained in:
AaronLiu
2025-04-20 17:31:25 +08:00
committed by GitHub
parent da4e44b77a
commit 21d158db07
597 changed files with 119415 additions and 41692 deletions

View File

@@ -0,0 +1,183 @@
package qiniu
import (
"context"
"encoding/json"
"fmt"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/mediameta"
"github.com/cloudreve/Cloudreve/v4/pkg/request"
"github.com/samber/lo"
"math"
"net/http"
"strconv"
"strings"
"time"
)
const (
exifParam = "exif"
avInfoParam = "avinfo"
mediaInfoTTL = time.Duration(10) * time.Minute
)
var (
supportedImageExt = []string{"jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff"}
)
type (
ImageProp struct {
Value string `json:"val"`
}
ImageInfo map[string]ImageProp
QiniuMediaError struct {
Error string `json:"error"`
Code int `json:"code"`
}
)
func (handler *Driver) extractAvMeta(ctx context.Context, path string) ([]driver.MediaMeta, error) {
resp, err := handler.extractMediaInfo(ctx, path, avInfoParam)
if err != nil {
return nil, err
}
var avInfo *mediameta.FFProbeMeta
if err := json.Unmarshal([]byte(resp), &avInfo); err != nil {
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
}
metas := mediameta.ProbeMetaTransform(avInfo)
if artist, ok := avInfo.Format.Tags["artist"]; ok {
metas = append(metas, driver.MediaMeta{
Key: mediameta.Artist,
Value: artist,
Type: driver.MediaTypeMusic,
})
}
if album, ok := avInfo.Format.Tags["album"]; ok {
metas = append(metas, driver.MediaMeta{
Key: mediameta.MusicAlbum,
Value: album,
Type: driver.MediaTypeMusic,
})
}
if title, ok := avInfo.Format.Tags["title"]; ok {
metas = append(metas, driver.MediaMeta{
Key: mediameta.MusicTitle,
Value: title,
Type: driver.MediaTypeMusic,
})
}
return metas, nil
}
func (handler *Driver) extractImageMeta(ctx context.Context, path string) ([]driver.MediaMeta, error) {
resp, err := handler.extractMediaInfo(ctx, path, exifParam)
if err != nil {
return nil, err
}
var imageInfo ImageInfo
if err := json.Unmarshal([]byte(resp), &imageInfo); err != nil {
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
}
metas := make([]driver.MediaMeta, 0)
exifMap := lo.MapEntries(imageInfo, func(key string, value ImageProp) (string, string) {
return key, value.Value
})
metas = append(metas, mediameta.ExtractExifMap(exifMap, time.Time{})...)
metas = append(metas, parseGpsInfo(imageInfo)...)
for i := 0; i < len(metas); i++ {
metas[i].Type = driver.MetaTypeExif
}
return metas, nil
}
func (handler *Driver) extractMediaInfo(ctx context.Context, path string, param string) (string, error) {
mediaInfoExpire := time.Now().Add(mediaInfoTTL)
ediaInfoUrl := handler.signSourceURL(fmt.Sprintf("%s?%s", path, param), &mediaInfoExpire)
resp, err := handler.httpClient.
Request(http.MethodGet, ediaInfoUrl, nil, request.WithContext(ctx)).
CheckHTTPResponse(http.StatusOK).
GetResponseIgnoreErr()
if err != nil {
return "", unmarshalError(resp, err)
}
return resp, nil
}
func unmarshalError(resp string, originErr error) error {
if resp == "" {
return originErr
}
var err QiniuMediaError
if err := json.Unmarshal([]byte(resp), &err); err != nil {
return fmt.Errorf("failed to unmarshal qiniu error: %w", err)
}
return fmt.Errorf("qiniu error: %s", err.Error)
}
func parseGpsInfo(imageInfo ImageInfo) []driver.MediaMeta {
latitude := imageInfo["GPSLatitude"] // 31, 16.2680820, 0
longitude := imageInfo["GPSLongitude"] // 120, 42.9103939, 0
latRef := imageInfo["GPSLatitudeRef"] // N
lonRef := imageInfo["GPSLongitudeRef"] // E
// Make sure all value exist in map
if latitude.Value == "" || longitude.Value == "" || latRef.Value == "" || lonRef.Value == "" {
return nil
}
lat := parseRawGPS(latitude.Value, latRef.Value)
lon := parseRawGPS(longitude.Value, lonRef.Value)
if !math.IsNaN(lat) && !math.IsNaN(lon) {
lat, lng := mediameta.NormalizeGPS(lat, lon)
return []driver.MediaMeta{{
Key: mediameta.GpsLat,
Value: fmt.Sprintf("%f", lat),
}, {
Key: mediameta.GpsLng,
Value: fmt.Sprintf("%f", lng),
}}
}
return nil
}
func parseRawGPS(gpsStr string, ref string) float64 {
elem := strings.Split(gpsStr, ", ")
if len(elem) < 1 {
return 0
}
var (
deg float64
minutes float64
seconds float64
)
deg, _ = strconv.ParseFloat(elem[0], 64)
if len(elem) >= 2 {
minutes, _ = strconv.ParseFloat(elem[1], 64)
}
if len(elem) >= 3 {
seconds, _ = strconv.ParseFloat(elem[2], 64)
}
decimal := deg + minutes/60.0 + seconds/3600.0
if ref == "S" || ref == "W" {
return -decimal
}
return decimal
}

View File

@@ -0,0 +1,428 @@
package qiniu
import (
"context"
"encoding/base64"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
"github.com/cloudreve/Cloudreve/v4/pkg/cluster/routes"
"github.com/cloudreve/Cloudreve/v4/pkg/conf"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk/backoff"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/mime"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/request"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
"github.com/samber/lo"
"io"
"net/http"
"net/url"
"os"
"time"
)
const (
chunkRetrySleep = time.Duration(5) * time.Second
maxDeleteBatch = 1000
trafficLimitParam = "X-Qiniu-Traffic-Limit"
)
var (
features = &boolset.BooleanSet{}
)
// Driver 本地策略适配器
type Driver struct {
policy *ent.StoragePolicy
mac *qbox.Mac
cfg *storage.Config
bucket *storage.BucketManager
settings setting.Provider
l logging.Logger
config conf.ConfigProvider
mime mime.MimeDetector
httpClient request.Client
chunkSize int64
}
func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provider,
config conf.ConfigProvider, l logging.Logger, mime mime.MimeDetector) (*Driver, error) {
chunkSize := policy.Settings.ChunkSize
if policy.Settings.ChunkSize == 0 {
chunkSize = 25 << 20 // 25 MB
}
mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
cfg := &storage.Config{UseHTTPS: true}
driver := &Driver{
policy: policy,
settings: settings,
chunkSize: chunkSize,
config: config,
l: l,
mime: mime,
mac: mac,
cfg: cfg,
bucket: storage.NewBucketManager(mac, cfg),
httpClient: request.NewClient(config, request.WithLogger(l)),
}
return driver, nil
}
//
//// List 列出给定路径下的文件
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// base = strings.TrimPrefix(base, "/")
// if base != "" {
// base += "/"
// }
//
// var (
// delimiter string
// marker string
// objects []storage.ListItem
// commons []string
// )
// if !recursive {
// delimiter = "/"
// }
//
// for {
// entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
// handler.policy.BucketName,
// base, delimiter, marker, 1000)
// if err != nil {
// return nil, err
// }
// objects = append(objects, entries...)
// commons = append(commons, folders...)
// if !hashNext {
// break
// }
// marker = nextMarker
// }
//
// // 处理列取结果
// res := make([]response.Object, 0, len(objects)+len(commons))
// // 处理目录
// for _, object := range commons {
// rel, err := filepath.Rel(base, object)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object),
// RelativePath: filepath.ToSlash(rel),
// Size: 0,
// IsDir: true,
// LastModify: time.Now(),
// })
// }
// // 处理文件
// for _, object := range objects {
// rel, err := filepath.Rel(base, object.Key)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object.Key),
// Source: object.Key,
// RelativePath: filepath.ToSlash(rel),
// Size: uint64(object.Fsize),
// IsDir: false,
// LastModify: time.Unix(object.PutTime/10000000, 0),
// })
// }
//
// return res, nil
//}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error {
defer file.Close()
// 凭证有效期
credentialTTL := handler.settings.UploadSessionTTL(ctx)
// 是否允许覆盖
overwrite := file.Mode&fs.ModeOverwrite == fs.ModeOverwrite
// 生成上传策略
scope := handler.policy.BucketName
if overwrite {
scope = fmt.Sprintf("%s:%s", handler.policy.BucketName, file.Props.SavePath)
}
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: scope,
SaveKey: file.Props.SavePath,
ForceSaveKey: true,
FsizeLimit: file.Props.Size,
Expires: uint64(time.Now().Add(credentialTTL).Unix()),
}
upToken := putPolicy.UploadToken(handler.mac)
// 初始化分片上传
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
upHost, err := resumeUploader.UpHost(handler.policy.AccessKey, handler.policy.BucketName)
if err != nil {
return fmt.Errorf("failed to get upload host: %w", err)
}
ret := &storage.InitPartsRet{}
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.policy.BucketName, file.Props.SavePath, true, ret)
if err != nil {
return fmt.Errorf("failed to initiate multipart upload: %w", err)
}
chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{
Max: handler.settings.ChunkRetryLimit(ctx),
Sleep: chunkRetrySleep,
}, handler.settings.UseChunkBuffer(ctx), handler.l, handler.settings.TempPath(ctx))
parts := make([]*storage.UploadPartsRet, 0, chunks.Num())
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
partRet := &storage.UploadPartsRet{}
err := resumeUploader.UploadParts(
ctx, upToken, upHost, handler.policy.BucketName, file.Props.SavePath, true, ret.UploadID,
int64(current.Index()+1), "", partRet, content, int(current.Length()))
if err == nil {
parts = append(parts, partRet)
}
return err
}
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
_ = handler.cancelUpload(upHost, file.Props.SavePath, ret.UploadID, upToken)
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
mimeType := file.Props.MimeType
if mimeType == "" {
handler.mime.TypeByName(file.Props.Uri.Name())
}
err = resumeUploader.CompleteParts(ctx, upToken, upHost, nil, handler.policy.BucketName,
file.Props.SavePath, true, ret.UploadID, &storage.RputV2Extra{
MimeType: mimeType,
Progresses: lo.Map(parts, func(part *storage.UploadPartsRet, i int) storage.UploadPartInfo {
return storage.UploadPartInfo{
Etag: part.Etag,
PartNumber: int64(i) + 1,
}
}),
})
if err != nil {
_ = handler.cancelUpload(upHost, file.Props.SavePath, ret.UploadID, upToken)
}
return nil
}
// Delete 删除一个或多个文件,
// 返回未删除的文件
func (handler *Driver) Delete(ctx context.Context, files ...string) ([]string, error) {
groups := lo.Chunk(files, maxDeleteBatch)
failed := make([]string, 0)
var lastError error
for index, group := range groups {
handler.l.Debug("Process delete group #%d: %v", index, group)
// 删除文件
rets, err := handler.bucket.BatchWithContext(ctx, handler.policy.BucketName, lo.Map(group, func(key string, index int) string {
return storage.URIDelete(handler.policy.BucketName, key)
}))
// 处理删除结果
if err != nil {
for k, ret := range rets {
if ret.Code != 200 && ret.Code != 612 {
failed = append(failed, group[k])
lastError = err
}
}
}
}
if len(failed) > 0 && lastError == nil {
lastError = fmt.Errorf("failed to delete files: %v", failed)
}
return failed, lastError
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, expire *time.Time, ext string, e fs.Entity) (string, error) {
w, h := handler.settings.ThumbSize(ctx)
thumb := fmt.Sprintf("%s?imageView2/1/w/%d/h/%d", e.Source(), w, h)
return handler.signSourceURL(
thumb,
expire,
), nil
}
// Source 获取外链URL
func (handler *Driver) Source(ctx context.Context, e fs.Entity, args *driver.GetSourceArgs) (string, error) {
path := e.Source()
query := url.Values{}
// 加入下载相关设置
if args.IsDownload {
query.Add("attname", args.DisplayName)
}
if args.Speed > 0 {
// Byte 转换为 bit
args.Speed *= 8
// Qiniu 对速度值有范围限制
if args.Speed < 819200 {
args.Speed = 819200
}
if args.Speed > 838860800 {
args.Speed = 838860800
}
query.Add(trafficLimitParam, fmt.Sprintf("%d", args.Speed))
}
if len(query) > 0 {
path = path + "?" + query.Encode()
}
// 取得原始文件地址
return handler.signSourceURL(path, args.Expire), nil
}
func (handler *Driver) signSourceURL(path string, expire *time.Time) string {
var sourceURL string
if handler.policy.IsPrivate {
deadline := time.Now().Add(time.Duration(24) * time.Hour * 365 * 20).Unix()
if expire != nil {
deadline = expire.Unix()
}
sourceURL = storage.MakePrivateURL(handler.mac, handler.policy.Settings.ProxyServer, path, deadline)
} else {
sourceURL = storage.MakePublicURL(handler.policy.Settings.ProxyServer, path)
}
return sourceURL
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, uploadSession *fs.UploadSession, file *fs.UploadRequest) (*fs.UploadCredential, error) {
// 生成回调地址
siteURL := handler.settings.SiteURL(setting.UseFirstSiteUrl(ctx))
apiUrl := routes.MasterSlaveCallbackUrl(siteURL, types.PolicyTypeQiniu, uploadSession.Props.UploadSessionID, uploadSession.CallbackSecret).String()
// 创建上传策略
putPolicy := storage.PutPolicy{
Scope: fmt.Sprintf("%s:%s", handler.policy.BucketName, file.Props.SavePath),
CallbackURL: apiUrl,
CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
SaveKey: file.Props.SavePath,
ForceSaveKey: true,
FsizeLimit: file.Props.Size,
Expires: uint64(file.Props.ExpireAt.Unix()),
}
// 初始化分片上传
upToken := putPolicy.UploadToken(handler.mac)
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
upHost, err := resumeUploader.UpHost(handler.policy.AccessKey, handler.policy.BucketName)
if err != nil {
return nil, fmt.Errorf("failed to get upload host: %w", err)
}
ret := &storage.InitPartsRet{}
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.policy.BucketName, file.Props.SavePath, true, ret)
if err != nil {
return nil, fmt.Errorf("failed to initiate multipart upload: %w", err)
}
mimeType := file.Props.MimeType
if mimeType == "" {
handler.mime.TypeByName(file.Props.Uri.Name())
}
uploadSession.UploadID = ret.UploadID
return &fs.UploadCredential{
UploadID: ret.UploadID,
UploadURLs: []string{getUploadUrl(upHost, handler.policy.BucketName, file.Props.SavePath, ret.UploadID)},
Credential: upToken,
SessionID: uploadSession.Props.UploadSessionID,
ChunkSize: handler.chunkSize,
MimeType: mimeType,
}, nil
}
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
return nil, errors.New("not implemented")
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *fs.UploadSession) error {
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
}
func (handler *Driver) CompleteUpload(ctx context.Context, session *fs.UploadSession) error {
return nil
}
func (handler *Driver) Capabilities() *driver.Capabilities {
mediaMetaExts := handler.policy.Settings.MediaMetaExts
if !handler.policy.Settings.NativeMediaProcessing {
mediaMetaExts = nil
}
return &driver.Capabilities{
StaticFeatures: features,
MediaMetaSupportedExts: mediaMetaExts,
MediaMetaProxy: handler.policy.Settings.MediaMetaGeneratorProxy,
ThumbSupportedExts: handler.policy.Settings.ThumbExts,
ThumbProxy: handler.policy.Settings.ThumbGeneratorProxy,
ThumbSupportAllExts: handler.policy.Settings.ThumbSupportAllExts,
ThumbMaxSize: handler.policy.Settings.ThumbMaxSize,
}
}
func (handler *Driver) MediaMeta(ctx context.Context, path, ext string) ([]driver.MediaMeta, error) {
if util.ContainsString(supportedImageExt, ext) {
return handler.extractImageMeta(ctx, path)
}
return handler.extractAvMeta(ctx, path)
}
func (handler *Driver) LocalPath(ctx context.Context, path string) string {
return ""
}
func (handler *Driver) cancelUpload(upHost, savePath, uploadId, upToken string) error {
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
uploadUrl := getUploadUrl(upHost, handler.policy.BucketName, savePath, uploadId)
err := resumeUploader.Client.CallWith(context.Background(), nil, "DELETE", uploadUrl, http.Header{"Authorization": {"UpToken " + upToken}}, nil, 0)
if err != nil {
handler.l.Error("Failed to cancel upload session for %q: %s", savePath, err)
}
return err
}
func getUploadUrl(upHost, bucket, key, uploadId string) string {
return upHost + "/buckets/" + bucket + "/objects/" + base64.URLEncoding.EncodeToString([]byte(key)) + "/uploads/" + uploadId
}