Init V4 community edition (#2265)
* Init V4 community edition * Init V4 community edition
This commit is contained in:
AaronLiu
122
pkg/filemanager/driver/oss/callback.go
Normal file
122
pkg/filemanager/driver/oss/callback.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto"
|
||||
"crypto/md5"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
pubKeyHeader = "x-oss-pub-key-url"
|
||||
pubKeyPrefix = "http://gosspublic.alicdn.com/"
|
||||
pubKeyPrefixHttps = "https://gosspublic.alicdn.com/"
|
||||
pubKeyCacheKey = "oss_public_key"
|
||||
)
|
||||
|
||||
// GetPublicKey 从回调请求或缓存中获取OSS的回调签名公钥
|
||||
func GetPublicKey(r *http.Request, kv cache.Driver, client request.Client) ([]byte, error) {
|
||||
var pubKey []byte
|
||||
|
||||
// 尝试从缓存中获取
|
||||
pub, exist := kv.Get(pubKeyCacheKey)
|
||||
if exist {
|
||||
return pub.([]byte), nil
|
||||
}
|
||||
|
||||
// 从请求中获取
|
||||
pubURL, err := base64.StdEncoding.DecodeString(r.Header.Get(pubKeyHeader))
|
||||
if err != nil {
|
||||
return pubKey, err
|
||||
}
|
||||
|
||||
// 确保这个 public key 是由 OSS 颁发的
|
||||
if !strings.HasPrefix(string(pubURL), pubKeyPrefix) &&
|
||||
!strings.HasPrefix(string(pubURL), pubKeyPrefixHttps) {
|
||||
return pubKey, errors.New("public key url invalid")
|
||||
}
|
||||
|
||||
// 获取公钥
|
||||
body, err := client.Request("GET", string(pubURL), nil).
|
||||
CheckHTTPResponse(200).
|
||||
GetResponse()
|
||||
if err != nil {
|
||||
return pubKey, err
|
||||
}
|
||||
|
||||
// 写入缓存
|
||||
_ = kv.Set(pubKeyCacheKey, []byte(body), 86400*7)
|
||||
|
||||
return []byte(body), nil
|
||||
}
|
||||
|
||||
func getRequestMD5(r *http.Request) ([]byte, error) {
|
||||
var byteMD5 []byte
|
||||
|
||||
// 获取请求正文
|
||||
body, err := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if err != nil {
|
||||
return byteMD5, err
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
|
||||
strURLPathDecode, err := url.PathUnescape(r.URL.Path)
|
||||
if err != nil {
|
||||
return byteMD5, err
|
||||
}
|
||||
|
||||
strAuth := fmt.Sprintf("%s\n%s", strURLPathDecode, string(body))
|
||||
md5Ctx := md5.New()
|
||||
md5Ctx.Write([]byte(strAuth))
|
||||
byteMD5 = md5Ctx.Sum(nil)
|
||||
|
||||
return byteMD5, nil
|
||||
}
|
||||
|
||||
// VerifyCallbackSignature 验证OSS回调请求
|
||||
func VerifyCallbackSignature(r *http.Request, kv cache.Driver, client request.Client) error {
|
||||
bytePublicKey, err := GetPublicKey(r, kv, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
byteMD5, err := getRequestMD5(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
strAuthorizationBase64 := r.Header.Get("authorization")
|
||||
if strAuthorizationBase64 == "" {
|
||||
return errors.New("no authorization field in Request header")
|
||||
}
|
||||
authorization, _ := base64.StdEncoding.DecodeString(strAuthorizationBase64)
|
||||
|
||||
pubBlock, _ := pem.Decode(bytePublicKey)
|
||||
if pubBlock == nil {
|
||||
return errors.New("pubBlock not exist")
|
||||
}
|
||||
pubInterface, err := x509.ParsePKIXPublicKey(pubBlock.Bytes)
|
||||
if (pubInterface == nil) || (err != nil) {
|
||||
return err
|
||||
}
|
||||
pub := pubInterface.(*rsa.PublicKey)
|
||||
|
||||
errorVerifyPKCS1v15 := rsa.VerifyPKCS1v15(pub, crypto.MD5, byteMD5, authorization)
|
||||
if errorVerifyPKCS1v15 != nil {
|
||||
return errorVerifyPKCS1v15
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
359
pkg/filemanager/driver/oss/media.go
Normal file
359
pkg/filemanager/driver/oss/media.go
Normal file
@@ -0,0 +1,359 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/mediameta"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"github.com/samber/lo"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
imageInfoProcess = "image/info"
|
||||
videoInfoProcess = "video/info"
|
||||
audioInfoProcess = "audio/info"
|
||||
mediaInfoTTL = time.Duration(10) * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
supportedImageExt = []string{"jpg", "jpeg", "png", "gif", "bmp", "webp", "tiff", "heic", "heif"}
|
||||
supportedAudioExt = []string{"mp3", "wav", "flac", "aac", "m4a", "ogg", "wma", "ape", "alac", "amr", "opus"}
|
||||
supportedVideoExt = []string{"mp4", "mkv", "avi", "mov", "flv", "wmv", "rmvb", "webm", "3gp", "mpg", "mpeg", "m4v", "ts", "m3u8", "vob", "f4v", "rm", "asf", "divx", "ogv", "dat", "mts", "m2ts", "swf", "avi", "3g2", "m2v", "m4p", "m4b", "m4r", "m4v", "m4a"}
|
||||
)
|
||||
|
||||
type (
|
||||
ImageProp struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
ImageInfo map[string]ImageProp
|
||||
|
||||
Error struct {
|
||||
XMLName xml.Name `xml:"Error"`
|
||||
Text string `xml:",chardata"`
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
RequestId string `xml:"RequestId"`
|
||||
HostId string `xml:"HostId"`
|
||||
EC string `xml:"EC"`
|
||||
RecommendDoc string `xml:"RecommendDoc"`
|
||||
}
|
||||
|
||||
StreamMediaInfo struct {
|
||||
RequestID string `json:"RequestId"`
|
||||
Language string `json:"Language"`
|
||||
Title string `json:"Title"`
|
||||
VideoStreams []VideoStream `json:"VideoStreams"`
|
||||
AudioStreams []AudioStream `json:"AudioStreams"`
|
||||
Subtitles []Subtitle `json:"Subtitles"`
|
||||
StreamCount int64 `json:"StreamCount"`
|
||||
ProgramCount int64 `json:"ProgramCount"`
|
||||
FormatName string `json:"FormatName"`
|
||||
FormatLongName string `json:"FormatLongName"`
|
||||
Size int64 `json:"Size"`
|
||||
StartTime float64 `json:"StartTime"`
|
||||
Bitrate int64 `json:"Bitrate"`
|
||||
Artist string `json:"Artist"`
|
||||
AlbumArtist string `json:"AlbumArtist"`
|
||||
Composer string `json:"Composer"`
|
||||
Performer string `json:"Performer"`
|
||||
Album string `json:"Album"`
|
||||
Duration float64 `json:"Duration"`
|
||||
ProduceTime string `json:"ProduceTime"`
|
||||
LatLong string `json:"LatLong"`
|
||||
VideoWidth int64 `json:"VideoWidth"`
|
||||
VideoHeight int64 `json:"VideoHeight"`
|
||||
Addresses []Address `json:"Addresses"`
|
||||
}
|
||||
|
||||
Address struct {
|
||||
Language string `json:"Language"`
|
||||
AddressLine string `json:"AddressLine"`
|
||||
Country string `json:"Country"`
|
||||
Province string `json:"Province"`
|
||||
City string `json:"City"`
|
||||
District string `json:"District"`
|
||||
Township string `json:"Township"`
|
||||
}
|
||||
|
||||
AudioStream struct {
|
||||
Index int `json:"Index"`
|
||||
Language string `json:"Language"`
|
||||
CodecName string `json:"CodecName"`
|
||||
CodecLongName string `json:"CodecLongName"`
|
||||
CodecTimeBase string `json:"CodecTimeBase"`
|
||||
CodecTagString string `json:"CodecTagString"`
|
||||
CodecTag string `json:"CodecTag"`
|
||||
TimeBase string `json:"TimeBase"`
|
||||
StartTime float64 `json:"StartTime"`
|
||||
Duration float64 `json:"Duration"`
|
||||
Bitrate int64 `json:"Bitrate"`
|
||||
FrameCount int64 `json:"FrameCount"`
|
||||
Lyric string `json:"Lyric"`
|
||||
SampleFormat string `json:"SampleFormat"`
|
||||
SampleRate int64 `json:"SampleRate"`
|
||||
Channels int64 `json:"Channels"`
|
||||
ChannelLayout string `json:"ChannelLayout"`
|
||||
}
|
||||
|
||||
Subtitle struct {
|
||||
Index int64 `json:"Index"`
|
||||
Language string `json:"Language"`
|
||||
CodecName string `json:"CodecName"`
|
||||
CodecLongName string `json:"CodecLongName"`
|
||||
CodecTagString string `json:"CodecTagString"`
|
||||
CodecTag string `json:"CodecTag"`
|
||||
StartTime float64 `json:"StartTime"`
|
||||
Duration float64 `json:"Duration"`
|
||||
Bitrate int64 `json:"Bitrate"`
|
||||
Content string `json:"Content"`
|
||||
Width int64 `json:"Width"`
|
||||
Height int64 `json:"Height"`
|
||||
}
|
||||
|
||||
VideoStream struct {
|
||||
Index int `json:"Index"`
|
||||
Language string `json:"Language"`
|
||||
CodecName string `json:"CodecName"`
|
||||
CodecLongName string `json:"CodecLongName"`
|
||||
Profile string `json:"Profile"`
|
||||
CodecTimeBase string `json:"CodecTimeBase"`
|
||||
CodecTagString string `json:"CodecTagString"`
|
||||
CodecTag string `json:"CodecTag"`
|
||||
Width int `json:"Width"`
|
||||
Height int `json:"Height"`
|
||||
HasBFrames int `json:"HasBFrames"`
|
||||
SampleAspectRatio string `json:"SampleAspectRatio"`
|
||||
DisplayAspectRatio string `json:"DisplayAspectRatio"`
|
||||
PixelFormat string `json:"PixelFormat"`
|
||||
Level int `json:"Level"`
|
||||
FrameRate string `json:"FrameRate"`
|
||||
AverageFrameRate string `json:"AverageFrameRate"`
|
||||
TimeBase string `json:"TimeBase"`
|
||||
StartTime float64 `json:"StartTime"`
|
||||
Duration float64 `json:"Duration"`
|
||||
Bitrate int64 `json:"Bitrate"`
|
||||
FrameCount int64 `json:"FrameCount"`
|
||||
Rotate string `json:"Rotate"`
|
||||
BitDepth int `json:"BitDepth"`
|
||||
ColorSpace string `json:"ColorSpace"`
|
||||
ColorRange string `json:"ColorRange"`
|
||||
ColorTransfer string `json:"ColorTransfer"`
|
||||
ColorPrimaries string `json:"ColorPrimaries"`
|
||||
}
|
||||
)
|
||||
|
||||
func (handler *Driver) extractIMMMeta(ctx context.Context, path, category string) ([]driver.MediaMeta, error) {
|
||||
resp, err := handler.extractMediaInfo(ctx, path, category, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var info StreamMediaInfo
|
||||
if err := json.Unmarshal([]byte(resp), &info); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
|
||||
}
|
||||
|
||||
streams := lo.Map(info.VideoStreams, func(stream VideoStream, index int) mediameta.Stream {
|
||||
bitrate := ""
|
||||
if stream.Bitrate != 0 {
|
||||
bitrate = strconv.FormatInt(stream.Bitrate, 10)
|
||||
}
|
||||
return mediameta.Stream{
|
||||
Index: stream.Index,
|
||||
CodecName: stream.CodecName,
|
||||
CodecLongName: stream.CodecLongName,
|
||||
CodecType: "video",
|
||||
Width: stream.Width,
|
||||
Height: stream.Height,
|
||||
Duration: strconv.FormatFloat(stream.Duration, 'f', -1, 64),
|
||||
Bitrate: bitrate,
|
||||
}
|
||||
})
|
||||
streams = append(streams, lo.Map(info.AudioStreams, func(stream AudioStream, index int) mediameta.Stream {
|
||||
bitrate := ""
|
||||
if stream.Bitrate != 0 {
|
||||
bitrate = strconv.FormatInt(stream.Bitrate, 10)
|
||||
}
|
||||
return mediameta.Stream{
|
||||
Index: stream.Index,
|
||||
CodecName: stream.CodecName,
|
||||
CodecLongName: stream.CodecLongName,
|
||||
CodecType: "audio",
|
||||
Duration: strconv.FormatFloat(stream.Duration, 'f', -1, 64),
|
||||
Bitrate: bitrate,
|
||||
}
|
||||
})...)
|
||||
|
||||
metas := make([]driver.MediaMeta, 0)
|
||||
metas = append(metas, mediameta.ProbeMetaTransform(&mediameta.FFProbeMeta{
|
||||
Format: &mediameta.Format{
|
||||
FormatName: info.FormatName,
|
||||
FormatLongName: info.FormatLongName,
|
||||
Duration: strconv.FormatFloat(info.Duration, 'f', -1, 64),
|
||||
Bitrate: strconv.FormatInt(info.Bitrate, 10),
|
||||
},
|
||||
Streams: streams,
|
||||
})...)
|
||||
|
||||
if info.Artist != "" {
|
||||
metas = append(metas, driver.MediaMeta{
|
||||
Key: mediameta.MusicArtist,
|
||||
Value: info.Artist,
|
||||
Type: driver.MediaTypeMusic,
|
||||
})
|
||||
}
|
||||
|
||||
if info.AlbumArtist != "" {
|
||||
metas = append(metas, driver.MediaMeta{
|
||||
Key: mediameta.MusicAlbumArtists,
|
||||
Value: info.AlbumArtist,
|
||||
Type: driver.MediaTypeMusic,
|
||||
})
|
||||
}
|
||||
|
||||
if info.Composer != "" {
|
||||
metas = append(metas, driver.MediaMeta{
|
||||
Key: mediameta.MusicComposer,
|
||||
Value: info.Composer,
|
||||
Type: driver.MediaTypeMusic,
|
||||
})
|
||||
}
|
||||
|
||||
if info.Album != "" {
|
||||
metas = append(metas, driver.MediaMeta{
|
||||
Key: mediameta.MusicAlbum,
|
||||
Value: info.Album,
|
||||
Type: driver.MediaTypeMusic,
|
||||
})
|
||||
}
|
||||
|
||||
return metas, nil
|
||||
}
|
||||
|
||||
func (handler *Driver) extractImageMeta(ctx context.Context, path string) ([]driver.MediaMeta, error) {
|
||||
resp, err := handler.extractMediaInfo(ctx, path, imageInfoProcess, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var imageInfo ImageInfo
|
||||
if err := json.Unmarshal([]byte(resp), &imageInfo); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal media info: %w", err)
|
||||
}
|
||||
|
||||
metas := make([]driver.MediaMeta, 0)
|
||||
exifMap := lo.MapEntries(imageInfo, func(key string, value ImageProp) (string, string) {
|
||||
return key, value.Value
|
||||
})
|
||||
metas = append(metas, mediameta.ExtractExifMap(exifMap, time.Time{})...)
|
||||
metas = append(metas, parseGpsInfo(imageInfo)...)
|
||||
for i := 0; i < len(metas); i++ {
|
||||
metas[i].Type = driver.MetaTypeExif
|
||||
}
|
||||
|
||||
return metas, nil
|
||||
}
|
||||
|
||||
// extractMediaInfo Sends API calls to OSS IMM service to extract media info.
|
||||
func (handler *Driver) extractMediaInfo(ctx context.Context, path string, category string, forceSign bool) (string, error) {
|
||||
mediaOption := []oss.Option{oss.Process(category)}
|
||||
mediaInfoExpire := time.Now().Add(mediaInfoTTL)
|
||||
thumbURL, err := handler.signSourceURL(
|
||||
ctx,
|
||||
path,
|
||||
&mediaInfoExpire,
|
||||
mediaOption,
|
||||
forceSign,
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to sign media info url: %w", err)
|
||||
}
|
||||
|
||||
resp, err := handler.httpClient.
|
||||
Request(http.MethodGet, thumbURL, nil, request.WithContext(ctx)).
|
||||
CheckHTTPResponse(http.StatusOK).
|
||||
GetResponseIgnoreErr()
|
||||
if err != nil {
|
||||
return "", handleOssError(resp, err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func parseGpsInfo(imageInfo ImageInfo) []driver.MediaMeta {
|
||||
latitude := imageInfo["GPSLatitude"] // 31deg 16.26808'
|
||||
longitude := imageInfo["GPSLongitude"] // 120deg 42.91039'
|
||||
latRef := imageInfo["GPSLatitudeRef"] // North
|
||||
lonRef := imageInfo["GPSLongitudeRef"] // East
|
||||
|
||||
// Make sure all value exist in map
|
||||
if latitude.Value == "" || longitude.Value == "" || latRef.Value == "" || lonRef.Value == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
lat := parseRawGPS(latitude.Value, latRef.Value)
|
||||
lon := parseRawGPS(longitude.Value, lonRef.Value)
|
||||
if !math.IsNaN(lat) && !math.IsNaN(lon) {
|
||||
lat, lng := mediameta.NormalizeGPS(lat, lon)
|
||||
return []driver.MediaMeta{{
|
||||
Key: mediameta.GpsLat,
|
||||
Value: fmt.Sprintf("%f", lat),
|
||||
}, {
|
||||
Key: mediameta.GpsLng,
|
||||
Value: fmt.Sprintf("%f", lng),
|
||||
}}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseRawGPS(gpsStr string, ref string) float64 {
|
||||
elem := strings.Split(gpsStr, " ")
|
||||
if len(elem) < 1 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var (
|
||||
deg float64
|
||||
minutes float64
|
||||
seconds float64
|
||||
)
|
||||
|
||||
deg, _ = strconv.ParseFloat(strings.TrimSuffix(elem[0], "deg"), 64)
|
||||
if len(elem) >= 2 {
|
||||
minutes, _ = strconv.ParseFloat(strings.TrimSuffix(elem[1], "'"), 64)
|
||||
}
|
||||
if len(elem) >= 3 {
|
||||
seconds, _ = strconv.ParseFloat(strings.TrimSuffix(elem[2], "\""), 64)
|
||||
}
|
||||
|
||||
decimal := deg + minutes/60.0 + seconds/3600.0
|
||||
|
||||
if ref == "South" || ref == "West" {
|
||||
return -decimal
|
||||
}
|
||||
|
||||
return decimal
|
||||
}
|
||||
|
||||
func handleOssError(resp string, originErr error) error {
|
||||
if resp == "" {
|
||||
return originErr
|
||||
}
|
||||
|
||||
var err Error
|
||||
if err := xml.Unmarshal([]byte(resp), &err); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal oss error: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("oss error: %s", err.Message)
|
||||
}
|
||||
548
pkg/filemanager/driver/oss/oss.go
Normal file
548
pkg/filemanager/driver/oss/oss.go
Normal file
@@ -0,0 +1,548 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/cloudreve/Cloudreve/v4/ent"
|
||||
"github.com/cloudreve/Cloudreve/v4/inventory/types"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/cluster/routes"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/chunk/backoff"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/mime"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/util"
|
||||
"github.com/samber/lo"
|
||||
)
|
||||
|
||||
// UploadPolicy 阿里云OSS上传策略
|
||||
type UploadPolicy struct {
|
||||
Expiration string `json:"expiration"`
|
||||
Conditions []interface{} `json:"conditions"`
|
||||
}
|
||||
|
||||
// CallbackPolicy 回调策略
|
||||
type CallbackPolicy struct {
|
||||
CallbackURL string `json:"callbackUrl"`
|
||||
CallbackBody string `json:"callbackBody"`
|
||||
CallbackBodyType string `json:"callbackBodyType"`
|
||||
CallbackSNI bool `json:"callbackSNI"`
|
||||
}
|
||||
|
||||
// Driver 阿里云OSS策略适配器
|
||||
type Driver struct {
|
||||
policy *ent.StoragePolicy
|
||||
|
||||
client *oss.Client
|
||||
bucket *oss.Bucket
|
||||
settings setting.Provider
|
||||
l logging.Logger
|
||||
config conf.ConfigProvider
|
||||
mime mime.MimeDetector
|
||||
httpClient request.Client
|
||||
|
||||
chunkSize int64
|
||||
}
|
||||
|
||||
type key int
|
||||
|
||||
const (
|
||||
chunkRetrySleep = time.Duration(5) * time.Second
|
||||
uploadIdParam = "uploadId"
|
||||
partNumberParam = "partNumber"
|
||||
callbackParam = "callback"
|
||||
completeAllHeader = "x-oss-complete-all"
|
||||
maxDeleteBatch = 1000
|
||||
|
||||
// MultiPartUploadThreshold 服务端使用分片上传的阈值
|
||||
MultiPartUploadThreshold int64 = 5 * (1 << 30) // 5GB
|
||||
)
|
||||
|
||||
var (
|
||||
features = &boolset.BooleanSet{}
|
||||
)
|
||||
|
||||
func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provider,
|
||||
config conf.ConfigProvider, l logging.Logger, mime mime.MimeDetector) (*Driver, error) {
|
||||
chunkSize := policy.Settings.ChunkSize
|
||||
if policy.Settings.ChunkSize == 0 {
|
||||
chunkSize = 25 << 20 // 25 MB
|
||||
}
|
||||
|
||||
driver := &Driver{
|
||||
policy: policy,
|
||||
settings: settings,
|
||||
chunkSize: chunkSize,
|
||||
config: config,
|
||||
l: l,
|
||||
mime: mime,
|
||||
httpClient: request.NewClient(config, request.WithLogger(l)),
|
||||
}
|
||||
|
||||
return driver, driver.InitOSSClient(false)
|
||||
}
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler *Driver) CORS() error {
|
||||
return handler.client.SetBucketCORS(handler.policy.BucketName, []oss.CORSRule{
|
||||
{
|
||||
AllowedOrigin: []string{"*"},
|
||||
AllowedMethod: []string{
|
||||
"GET",
|
||||
"POST",
|
||||
"PUT",
|
||||
"DELETE",
|
||||
"HEAD",
|
||||
},
|
||||
ExposeHeader: []string{},
|
||||
AllowedHeader: []string{"*"},
|
||||
MaxAgeSeconds: 3600,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// InitOSSClient 初始化OSS鉴权客户端
|
||||
func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
|
||||
if handler.policy == nil {
|
||||
return errors.New("empty policy")
|
||||
}
|
||||
|
||||
opt := make([]oss.ClientOption, 0)
|
||||
|
||||
// 决定是否使用内网 Endpoint
|
||||
endpoint := handler.policy.Server
|
||||
if handler.policy.Settings.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
|
||||
endpoint = handler.policy.Settings.ServerSideEndpoint
|
||||
} else if handler.policy.Settings.UseCname {
|
||||
opt = append(opt, oss.UseCname(true))
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
|
||||
// 初始化客户端
|
||||
client, err := oss.New(endpoint, handler.policy.AccessKey, handler.policy.SecretKey, opt...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.client = client
|
||||
|
||||
// 初始化存储桶
|
||||
bucket, err := client.Bucket(handler.policy.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handler.bucket = bucket
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//// List 列出OSS上的文件
|
||||
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// // 列取文件
|
||||
// base = strings.TrimPrefix(base, "/")
|
||||
// if base != "" {
|
||||
// base += "/"
|
||||
// }
|
||||
//
|
||||
// var (
|
||||
// delimiter string
|
||||
// marker string
|
||||
// objects []oss.ObjectProperties
|
||||
// commons []string
|
||||
// )
|
||||
// if !recursive {
|
||||
// delimiter = "/"
|
||||
// }
|
||||
//
|
||||
// for {
|
||||
// subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
|
||||
// oss.MaxKeys(1000), oss.Delimiter(delimiter))
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// objects = append(objects, subRes.Objects...)
|
||||
// commons = append(commons, subRes.CommonPrefixes...)
|
||||
// marker = subRes.NextMarker
|
||||
// if marker == "" {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // 处理列取结果
|
||||
// res := make([]response.Object, 0, len(objects)+len(commons))
|
||||
// // 处理目录
|
||||
// for _, object := range commons {
|
||||
// rel, err := filepath.Rel(base, object)
|
||||
// if err != nil {
|
||||
// continue
|
||||
// }
|
||||
// res = append(res, response.Object{
|
||||
// Name: path.Base(object),
|
||||
// RelativePath: filepath.ToSlash(rel),
|
||||
// Size: 0,
|
||||
// IsDir: true,
|
||||
// LastModify: time.Now(),
|
||||
// })
|
||||
// }
|
||||
// // 处理文件
|
||||
// for _, object := range objects {
|
||||
// rel, err := filepath.Rel(base, object.Key)
|
||||
// if err != nil {
|
||||
// continue
|
||||
// }
|
||||
// res = append(res, response.Object{
|
||||
// Name: path.Base(object.Key),
|
||||
// Source: object.Key,
|
||||
// RelativePath: filepath.ToSlash(rel),
|
||||
// Size: uint64(object.Size),
|
||||
// IsDir: false,
|
||||
// LastModify: object.LastModified,
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// return res, nil
|
||||
//}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error {
|
||||
defer file.Close()
|
||||
|
||||
// 凭证有效期
|
||||
credentialTTL := handler.settings.UploadSessionTTL(ctx)
|
||||
|
||||
mimeType := file.Props.MimeType
|
||||
if mimeType == "" {
|
||||
handler.mime.TypeByName(file.Props.Uri.Name())
|
||||
}
|
||||
|
||||
// 是否允许覆盖
|
||||
overwrite := file.Mode&fs.ModeOverwrite == fs.ModeOverwrite
|
||||
options := []oss.Option{
|
||||
oss.WithContext(ctx),
|
||||
oss.Expires(time.Now().Add(credentialTTL * time.Second)),
|
||||
oss.ForbidOverWrite(!overwrite),
|
||||
oss.ContentType(mimeType),
|
||||
}
|
||||
|
||||
// 小文件直接上传
|
||||
if file.Props.Size < MultiPartUploadThreshold {
|
||||
return handler.bucket.PutObject(file.Props.SavePath, file, options...)
|
||||
}
|
||||
|
||||
// 超过阈值时使用分片上传
|
||||
imur, err := handler.bucket.InitiateMultipartUpload(file.Props.SavePath, options...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initiate multipart upload: %w", err)
|
||||
}
|
||||
|
||||
parts := make([]oss.UploadPart, 0)
|
||||
|
||||
chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{
|
||||
Max: handler.settings.ChunkRetryLimit(ctx),
|
||||
Sleep: chunkRetrySleep,
|
||||
}, handler.settings.UseChunkBuffer(ctx), handler.l, handler.settings.TempPath(ctx))
|
||||
|
||||
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
|
||||
part, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1, oss.WithContext(ctx))
|
||||
if err == nil {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for chunks.Next() {
|
||||
if err := chunks.Process(uploadFunc); err != nil {
|
||||
handler.cancelUpload(imur)
|
||||
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = handler.bucket.CompleteMultipartUpload(imur, parts, oss.ForbidOverWrite(!overwrite), oss.WithContext(ctx))
|
||||
if err != nil {
|
||||
handler.cancelUpload(imur)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件，
|
||||
// 返回未删除的文件
|
||||
func (handler *Driver) Delete(ctx context.Context, files ...string) ([]string, error) {
|
||||
groups := lo.Chunk(files, maxDeleteBatch)
|
||||
failed := make([]string, 0)
|
||||
var lastError error
|
||||
for index, group := range groups {
|
||||
handler.l.Debug("Process delete group #%d: %v", index, group)
|
||||
// 删除文件
|
||||
delRes, err := handler.bucket.DeleteObjects(group)
|
||||
if err != nil {
|
||||
failed = append(failed, group...)
|
||||
lastError = err
|
||||
continue
|
||||
}
|
||||
|
||||
// 统计未删除的文件
|
||||
failed = append(failed, util.SliceDifference(files, delRes.DeletedObjects)...)
|
||||
}
|
||||
|
||||
if len(failed) > 0 && lastError == nil {
|
||||
lastError = fmt.Errorf("failed to delete files: %v", failed)
|
||||
}
|
||||
|
||||
return failed, lastError
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler *Driver) Thumb(ctx context.Context, expire *time.Time, ext string, e fs.Entity) (string, error) {
|
||||
usePublicEndpoint := true
|
||||
if forceUsePublicEndpoint, ok := ctx.Value(driver.ForceUsePublicEndpointCtx{}).(bool); ok {
|
||||
usePublicEndpoint = forceUsePublicEndpoint
|
||||
}
|
||||
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(usePublicEndpoint); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
w, h := handler.settings.ThumbSize(ctx)
|
||||
thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d", h, w)
|
||||
thumbOption := []oss.Option{oss.Process(thumbParam)}
|
||||
thumbURL, err := handler.signSourceURL(
|
||||
ctx,
|
||||
e.Source(),
|
||||
expire,
|
||||
thumbOption,
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return thumbURL, nil
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler *Driver) Source(ctx context.Context, e fs.Entity, args *driver.GetSourceArgs) (string, error) {
|
||||
// 初始化客户端
|
||||
usePublicEndpoint := true
|
||||
if forceUsePublicEndpoint, ok := ctx.Value(driver.ForceUsePublicEndpointCtx{}).(bool); ok {
|
||||
usePublicEndpoint = forceUsePublicEndpoint
|
||||
}
|
||||
if err := handler.InitOSSClient(usePublicEndpoint); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 添加各项设置
|
||||
var signOptions = make([]oss.Option, 0, 2)
|
||||
if args.IsDownload {
|
||||
encodedFilename := url.PathEscape(args.DisplayName)
|
||||
signOptions = append(signOptions, oss.ResponseContentDisposition(fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`,
|
||||
encodedFilename, encodedFilename)))
|
||||
}
|
||||
if args.Speed > 0 {
|
||||
// Byte 转换为 bit
|
||||
args.Speed *= 8
|
||||
|
||||
// OSS对速度值有范围限制
|
||||
if args.Speed < 819200 {
|
||||
args.Speed = 819200
|
||||
}
|
||||
if args.Speed > 838860800 {
|
||||
args.Speed = 838860800
|
||||
}
|
||||
signOptions = append(signOptions, oss.TrafficLimitParam(args.Speed))
|
||||
}
|
||||
|
||||
return handler.signSourceURL(ctx, e.Source(), args.Expire, signOptions, false)
|
||||
}
|
||||
|
||||
func (handler *Driver) signSourceURL(ctx context.Context, path string, expire *time.Time, options []oss.Option, forceSign bool) (string, error) {
|
||||
ttl := int64(86400 * 365 * 20)
|
||||
if expire != nil {
|
||||
ttl = int64(time.Until(*expire).Seconds())
|
||||
}
|
||||
|
||||
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 将最终生成的签名URL域名换成用户自定义的加速域名（如果有）
|
||||
finalURL, err := url.Parse(signedURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 公有空间替换掉Key及不支持的头
|
||||
if !handler.policy.IsPrivate && !forceSign {
|
||||
query := finalURL.Query()
|
||||
query.Del("OSSAccessKeyId")
|
||||
query.Del("Signature")
|
||||
query.Del("response-content-disposition")
|
||||
query.Del("x-oss-traffic-limit")
|
||||
finalURL.RawQuery = query.Encode()
|
||||
}
|
||||
return finalURL.String(), nil
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler *Driver) Token(ctx context.Context, uploadSession *fs.UploadSession, file *fs.UploadRequest) (*fs.UploadCredential, error) {
|
||||
// 初始化客户端
|
||||
if err := handler.InitOSSClient(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 生成回调地址
|
||||
siteURL := handler.settings.SiteURL(setting.UseFirstSiteUrl(ctx))
|
||||
// 在从机端创建上传会话
|
||||
uploadSession.ChunkSize = handler.chunkSize
|
||||
uploadSession.Callback = routes.MasterSlaveCallbackUrl(siteURL, types.PolicyTypeOss, uploadSession.Props.UploadSessionID, uploadSession.CallbackSecret).String()
|
||||
|
||||
// 回调策略
|
||||
callbackPolicy := CallbackPolicy{
|
||||
CallbackURL: uploadSession.Callback,
|
||||
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
|
||||
CallbackBodyType: "application/json",
|
||||
CallbackSNI: true,
|
||||
}
|
||||
callbackPolicyJSON, err := json.Marshal(callbackPolicy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to encode callback policy: %w", err)
|
||||
}
|
||||
callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
|
||||
|
||||
mimeType := file.Props.MimeType
|
||||
if mimeType == "" {
|
||||
handler.mime.TypeByName(file.Props.Uri.Name())
|
||||
}
|
||||
|
||||
// 初始化分片上传
|
||||
options := []oss.Option{
|
||||
oss.WithContext(ctx),
|
||||
oss.Expires(uploadSession.Props.ExpireAt),
|
||||
oss.ForbidOverWrite(true),
|
||||
oss.ContentType(mimeType),
|
||||
}
|
||||
imur, err := handler.bucket.InitiateMultipartUpload(file.Props.SavePath, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
|
||||
}
|
||||
uploadSession.UploadID = imur.UploadID
|
||||
|
||||
// 为每个分片签名上传 URL
|
||||
chunks := chunk.NewChunkGroup(file, handler.chunkSize, &backoff.ConstantBackoff{}, false, handler.l, "")
|
||||
urls := make([]string, chunks.Num())
|
||||
ttl := int64(time.Until(uploadSession.Props.ExpireAt).Seconds())
|
||||
for chunks.Next() {
|
||||
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
|
||||
signedURL, err := handler.bucket.SignURL(file.Props.SavePath, oss.HTTPPut,
|
||||
ttl,
|
||||
oss.AddParam(partNumberParam, strconv.Itoa(c.Index()+1)),
|
||||
oss.AddParam(uploadIdParam, imur.UploadID),
|
||||
oss.ContentType("application/octet-stream"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
urls[c.Index()] = signedURL
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// 签名完成分片上传的URL
|
||||
completeURL, err := handler.bucket.SignURL(file.Props.SavePath, oss.HTTPPost, ttl,
|
||||
oss.ContentType("application/octet-stream"),
|
||||
oss.AddParam(uploadIdParam, imur.UploadID),
|
||||
oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
|
||||
oss.SetHeader(completeAllHeader, "yes"),
|
||||
oss.ForbidOverWrite(true),
|
||||
oss.AddParam(callbackParam, callbackPolicyEncoded))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &fs.UploadCredential{
|
||||
UploadID: imur.UploadID,
|
||||
UploadURLs: urls,
|
||||
CompleteURL: completeURL,
|
||||
SessionID: uploadSession.Props.UploadSessionID,
|
||||
ChunkSize: handler.chunkSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *fs.UploadSession) error {
|
||||
return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.Props.SavePath}, oss.WithContext(ctx))
|
||||
}
|
||||
|
||||
func (handler *Driver) CompleteUpload(ctx context.Context, session *fs.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (handler *Driver) Capabilities() *driver.Capabilities {
|
||||
mediaMetaExts := handler.policy.Settings.MediaMetaExts
|
||||
if !handler.policy.Settings.NativeMediaProcessing {
|
||||
mediaMetaExts = nil
|
||||
}
|
||||
return &driver.Capabilities{
|
||||
StaticFeatures: features,
|
||||
MediaMetaSupportedExts: mediaMetaExts,
|
||||
MediaMetaProxy: handler.policy.Settings.MediaMetaGeneratorProxy,
|
||||
ThumbSupportedExts: handler.policy.Settings.ThumbExts,
|
||||
ThumbProxy: handler.policy.Settings.ThumbGeneratorProxy,
|
||||
ThumbSupportAllExts: handler.policy.Settings.ThumbSupportAllExts,
|
||||
ThumbMaxSize: handler.policy.Settings.ThumbMaxSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *Driver) MediaMeta(ctx context.Context, path, ext string) ([]driver.MediaMeta, error) {
|
||||
if util.ContainsString(supportedImageExt, ext) {
|
||||
return handler.extractImageMeta(ctx, path)
|
||||
}
|
||||
|
||||
if util.ContainsString(supportedVideoExt, ext) {
|
||||
return handler.extractIMMMeta(ctx, path, videoInfoProcess)
|
||||
}
|
||||
|
||||
if util.ContainsString(supportedAudioExt, ext) {
|
||||
return handler.extractIMMMeta(ctx, path, audioInfoProcess)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported media type in oss: %s", ext)
|
||||
}
|
||||
|
||||
func (handler *Driver) LocalPath(ctx context.Context, path string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (handler *Driver) cancelUpload(imur oss.InitiateMultipartUploadResult) {
|
||||
if err := handler.bucket.AbortMultipartUpload(imur); err != nil {
|
||||
handler.l.Warning("failed to abort multipart upload: %s", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user