MiaoStars/cloudreve
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(workflow): import files from external storage

Browse Source
This commit is contained in:
Aaron Liu
2025-05-20 10:45:16 +08:00
parent 5d72faf688
commit a10a008ed7
32 changed files with 1071 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
pkg/filemanager/driver/cos/cos.go
View File

@@ -8,6 +8,9 @@ import (
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
@@ -126,80 +129,88 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
return driver, nil
}
//
//// List 列出COS文件
//func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// // 初始化列目录参数
// opt := &cossdk.BucketGetOptions{
// Prefix: strings.TrimPrefix(base, "/"),
// EncodingType: "",
// MaxKeys: 1000,
// }
// // 是否为递归列出
// if !recursive {
// opt.Delimiter = "/"
// }
// // 手动补齐结尾的slash
// if opt.Prefix != "" {
// opt.Prefix += "/"
// }
//
// var (
// marker string
// objects []cossdk.Object
// commons []string
// )
//
// for {
// res, _, err := handler.client.Bucket.Get(ctx, opt)
// if err != nil {
// return nil, err
// }
// objects = append(objects, res.Contents...)
// commons = append(commons, res.CommonPrefixes...)
// // 如果本次未列取完，则继续使用marker获取结果
// marker = res.NextMarker
// // marker 为空时结果列取完毕，跳出
// if marker == "" {
// break
// }
// }
//
// // 处理列取结果
// res := make([]response.Object, 0, len(objects)+len(commons))
// // 处理目录
// for _, object := range commons {
// rel, err := filepath.Rel(opt.Prefix, object)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object),
// RelativePath: filepath.ToSlash(rel),
// Size: 0,
// IsDir: true,
// LastModify: time.Now(),
// })
// }
// // 处理文件
// for _, object := range objects {
// rel, err := filepath.Rel(opt.Prefix, object.Key)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object.Key),
// Source: object.Key,
// RelativePath: filepath.ToSlash(rel),
// Size: uint64(object.Size),
// IsDir: false,
// LastModify: time.Now(),
// })
// }
//
// return res, nil
//
//}
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
// 初始化列目录参数
opt := &cossdk.BucketGetOptions{
Prefix: strings.TrimPrefix(base, "/"),
EncodingType: "",
MaxKeys: 1000,
}
// 是否为递归列出
if !recursive {
opt.Delimiter = "/"
}
// 手动补齐结尾的slash
if opt.Prefix != "" {
opt.Prefix += "/"
}
var (
marker string
objects []cossdk.Object
commons []string
)
for {
res, _, err := handler.client.Bucket.Get(ctx, opt)
if err != nil {
handler.l.Warning("Failed to list objects: %s", err)
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完，则继续使用marker获取结果
marker = res.NextMarker
// marker 为空时结果列取完毕，跳出
if marker == "" {
break
}
}
// 处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(opt.Prefix, object)
if err != nil {
handler.l.Warning("Failed to get relative path: %s", err)
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
onProgress(len(commons))
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(opt.Prefix, object.Key)
if err != nil {
handler.l.Warning("Failed to get relative path: %s", err)
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: object.Size,
IsDir: false,
LastModify: time.Now(),
})
}
onProgress(len(res))
return res, nil
}
// CORS 创建跨域策略
func (handler Driver) CORS() error {

9
pkg/filemanager/driver/handler.go
View File

@@ -2,6 +2,7 @@ package driver
import (
"context"
"encoding/gob"
"os"
"time"
@@ -76,7 +77,7 @@ type (
// List 递归列取远程端path路径下文件、目录，不包含path本身，
// 返回的对象路径以path作为起始根目录.
// recursive - 是否递归列出
// List(ctx context.Context, path string, recursive bool) ([]response.Object, error)
List(ctx context.Context, base string, onProgress ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error)
// Capabilities returns the capabilities of this handler
Capabilities() *Capabilities
@@ -108,6 +109,8 @@ type (
// BrowserRelayedDownload indicates whether to relay download via stream-saver.
BrowserRelayedDownload bool
}
ListProgressFunc func(int)
)
const (
@@ -122,3 +125,7 @@ type ForceUsePublicEndpointCtx struct{}
func WithForcePublicEndpoint(ctx context.Context, value bool) context.Context {
return context.WithValue(ctx, ForceUsePublicEndpointCtx{}, value)
}
func init() {
gob.Register(fs.PhysicalObject{})
}

92
pkg/filemanager/driver/local/local.go
View File

@@ -58,51 +58,53 @@ func New(p *ent.StoragePolicy, l logging.Logger, config conf.ConfigProvider) *Dr
}
}
//// List 递归列取给定物理路径下所有文件
//func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
// var res []response.Object
//
// // 取得起始路径
// root := util.RelativePath(filepath.FromSlash(path))
//
// // 开始遍历路径下的文件、目录
// err := filepath.Walk(root,
// func(path string, info os.FileInfo, err error) error {
// // 跳过根目录
// if path == root {
// return nil
// }
//
// if err != nil {
// util.Log().Warning("Failed to walk folder %q: %s", path, err)
// return filepath.SkipDir
// }
//
// // 将遍历对象的绝对路径转换为相对路径
// rel, err := filepath.Rel(root, path)
// if err != nil {
// return err
// }
//
// res = append(res, response.Object{
// Name: info.Name(),
// RelativePath: filepath.ToSlash(rel),
// Source: path,
// Size: uint64(info.Size()),
// IsDir: info.IsDir(),
// LastModify: info.ModTime(),
// })
//
// // 如果非递归，则不步入目录
// if !recursive && info.IsDir() {
// return filepath.SkipDir
// }
//
// return nil
// })
//
// return res, err
//}
func (handler *Driver) List(ctx context.Context, path string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
var res []fs.PhysicalObject
root := handler.LocalPath(ctx, path)
err := filepath.Walk(root,
func(path string, info os.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
// Skip root directory
if path == root {
return nil
}
if err != nil {
handler.l.Warning("Failed to walk folder %q: %s", path, err)
return filepath.SkipDir
}
// Transform absolute path to relative path
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
res = append(res, fs.PhysicalObject{
Name: info.Name(),
RelativePath: filepath.ToSlash(rel),
Source: path,
Size: info.Size(),
IsDir: info.IsDir(),
LastModify: info.ModTime(),
})
onProgress(1)
// If not recursive, do not enter directory
if !recursive && info.IsDir() {
return filepath.SkipDir
}
return nil
})
return res, err
}
// Get 获取文件内容
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {

86
pkg/filemanager/driver/obs/obs.go
View File

@@ -9,7 +9,10 @@ import (
"io"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
@@ -104,6 +107,89 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
return driver, nil
}
func (d *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
opt := &obs.ListObjectsInput{
ListObjsInput: obs.ListObjsInput{
Prefix: strings.TrimPrefix(base, "/"),
EncodingType: "",
MaxKeys: 1000,
},
Bucket: d.policy.BucketName,
}
if !recursive {
opt.Delimiter = "/"
}
if opt.Prefix != "" {
opt.Prefix += "/"
}
var (
marker string
objects []obs.Content
commons []string
)
for {
res, err := d.obs.ListObjects(opt, obs.WithRequestContext(ctx))
if err != nil {
d.l.Warning("Failed to list objects: %s", err)
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完，则继续使用marker获取结果
marker = res.NextMarker
// marker 为空时结果列取完毕，跳出
if marker == "" {
break
}
}
// 处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(opt.Prefix, object)
if err != nil {
d.l.Warning("Failed to get relative path: %s", err)
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
onProgress(len(commons))
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(opt.Prefix, object.Key)
if err != nil {
d.l.Warning("Failed to get relative path: %s", err)
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: object.Size,
IsDir: false,
LastModify: time.Now(),
})
}
onProgress(len(res))
return res, nil
}
func (d *Driver) Put(ctx context.Context, file *fs.UploadRequest) error {
defer file.Close()

111
pkg/filemanager/driver/onedrive/onedrive.go
View File

@@ -5,6 +5,8 @@ import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"
@@ -23,14 +25,17 @@ import (
)
// Driver OneDrive 适配器
type Driver struct {
policy *ent.StoragePolicy
client Client
settings setting.Provider
config conf.ConfigProvider
l logging.Logger
chunkSize int64
}
type (
Driver struct {
policy *ent.StoragePolicy
client Client
settings setting.Provider
config conf.ConfigProvider
l logging.Logger
chunkSize int64
}
ListPathRealRootCtx struct{}
)
var (
features = &boolset.BooleanSet{}
@@ -66,50 +71,52 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
}, nil
}
//// List 列取项目
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// base = strings.TrimPrefix(base, "/")
// // 列取子项目
// objects, _ := handler.client.ListChildren(ctx, base)
//
// // 获取真实的列取起始根目录
// rootPath := base
// if realBase, ok := ctx.Value(fsctx.PathCtx).(string); ok {
// rootPath = realBase
// } else {
// ctx = context.WithValue(ctx, fsctx.PathCtx, base)
// }
//
// // 整理结果
// res := make([]response.Object, 0, len(objects))
// for _, object := range objects {
// source := path.Join(base, object.Name)
// rel, err := filepath.Rel(rootPath, source)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: object.Name,
// RelativePath: filepath.ToSlash(rel),
// Source: source,
// Size: uint64(object.Size),
// IsDir: object.Folder != nil,
// LastModify: time.Now(),
// })
// }
//
// // 递归列取子目录
// if recursive {
// for _, object := range objects {
// if object.Folder != nil {
// sub, _ := handler.List(ctx, path.Join(base, object.Name), recursive)
// res = append(res, sub...)
// }
// }
// }
//
// return res, nil
//}
// List 列取项目
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
base = strings.TrimPrefix(base, "/")
// 列取子项目
objects, _ := handler.client.ListChildren(ctx, base)
// 获取真实的列取起始根目录
rootPath := base
if realBase, ok := ctx.Value(ListPathRealRootCtx{}).(string); ok {
rootPath = realBase
} else {
ctx = context.WithValue(ctx, ListPathRealRootCtx{}, base)
}
// 整理结果
res := make([]fs.PhysicalObject, 0, len(objects))
for _, object := range objects {
source := path.Join(base, object.Name)
rel, err := filepath.Rel(rootPath, source)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: object.Name,
RelativePath: filepath.ToSlash(rel),
Source: source,
Size: object.Size,
IsDir: object.Folder != nil,
LastModify: time.Now(),
})
}
onProgress(len(objects))
// 递归列取子目录
if recursive {
for _, object := range objects {
if object.Folder != nil {
sub, _ := handler.List(ctx, path.Join(base, object.Name), onProgress, recursive)
res = append(res, sub...)
}
}
}
return res, nil
}
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
return nil, errors.New("not implemented")

137
pkg/filemanager/driver/oss/oss.go
View File

@@ -9,6 +9,8 @@ import (
"io"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@@ -154,72 +156,75 @@ func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
return nil
}
//// List 列出OSS上的文件
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// // 列取文件
// base = strings.TrimPrefix(base, "/")
// if base != "" {
// base += "/"
// }
//
// var (
// delimiter string
// marker string
// objects []oss.ObjectProperties
// commons []string
// )
// if !recursive {
// delimiter = "/"
// }
//
// for {
// subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
// oss.MaxKeys(1000), oss.Delimiter(delimiter))
// if err != nil {
// return nil, err
// }
// objects = append(objects, subRes.Objects...)
// commons = append(commons, subRes.CommonPrefixes...)
// marker = subRes.NextMarker
// if marker == "" {
// break
// }
// }
//
// // 处理列取结果
// res := make([]response.Object, 0, len(objects)+len(commons))
// // 处理目录
// for _, object := range commons {
// rel, err := filepath.Rel(base, object)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object),
// RelativePath: filepath.ToSlash(rel),
// Size: 0,
// IsDir: true,
// LastModify: time.Now(),
// })
// }
// // 处理文件
// for _, object := range objects {
// rel, err := filepath.Rel(base, object.Key)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object.Key),
// Source: object.Key,
// RelativePath: filepath.ToSlash(rel),
// Size: uint64(object.Size),
// IsDir: false,
// LastModify: object.LastModified,
// })
// }
//
// return res, nil
//}
// List 列出OSS上的文件
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []oss.ObjectProperties
commons []string
)
if !recursive {
delimiter = "/"
}
for {
subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
oss.MaxKeys(1000), oss.Delimiter(delimiter))
if err != nil {
return nil, err
}
objects = append(objects, subRes.Objects...)
commons = append(commons, subRes.CommonPrefixes...)
marker = subRes.NextMarker
if marker == "" {
break
}
}
// 处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
onProgress(len(commons))
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: object.Size,
IsDir: false,
LastModify: object.LastModified,
})
}
onProgress(len(res))
return res, nil
}
// Get 获取文件
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {

150
pkg/filemanager/driver/qiniu/qiniu.go
View File

@@ -5,6 +5,15 @@ import (
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
@@ -22,11 +31,6 @@ import (
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
"github.com/samber/lo"
"io"
"net/http"
"net/url"
"os"
"time"
)
const (
@@ -81,73 +85,75 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
return driver, nil
}
//
//// List 列出给定路径下的文件
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// base = strings.TrimPrefix(base, "/")
// if base != "" {
// base += "/"
// }
//
// var (
// delimiter string
// marker string
// objects []storage.ListItem
// commons []string
// )
// if !recursive {
// delimiter = "/"
// }
//
// for {
// entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
// handler.policy.BucketName,
// base, delimiter, marker, 1000)
// if err != nil {
// return nil, err
// }
// objects = append(objects, entries...)
// commons = append(commons, folders...)
// if !hashNext {
// break
// }
// marker = nextMarker
// }
//
// // 处理列取结果
// res := make([]response.Object, 0, len(objects)+len(commons))
// // 处理目录
// for _, object := range commons {
// rel, err := filepath.Rel(base, object)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object),
// RelativePath: filepath.ToSlash(rel),
// Size: 0,
// IsDir: true,
// LastModify: time.Now(),
// })
// }
// // 处理文件
// for _, object := range objects {
// rel, err := filepath.Rel(base, object.Key)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(object.Key),
// Source: object.Key,
// RelativePath: filepath.ToSlash(rel),
// Size: uint64(object.Fsize),
// IsDir: false,
// LastModify: time.Unix(object.PutTime/10000000, 0),
// })
// }
//
// return res, nil
//}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []storage.ListItem
commons []string
)
if !recursive {
delimiter = "/"
}
for {
entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
handler.policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
return nil, err
}
objects = append(objects, entries...)
commons = append(commons, folders...)
if !hashNext {
break
}
marker = nextMarker
}
// 处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
onProgress(len(commons))
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: int64(object.Fsize),
IsDir: false,
LastModify: time.Unix(object.PutTime/10000000, 0),
})
}
onProgress(len(objects))
return res, nil
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file *fs.UploadRequest) error {

35
pkg/filemanager/driver/remote/client.go
View File

@@ -5,6 +5,12 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v4/application/constants"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/pkg/auth"
@@ -19,11 +25,6 @@ import (
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"github.com/gofrs/uuid"
"io"
"net/http"
"net/url"
"strings"
"time"
)
const (
@@ -45,6 +46,8 @@ type Client interface {
MediaMeta(ctx context.Context, src, ext string) ([]driver.MediaMeta, error)
// DeleteFiles deletes files from remote server
DeleteFiles(ctx context.Context, files ...string) ([]string, error)
// List lists files from remote server
List(ctx context.Context, path string, recursive bool) ([]fs.PhysicalObject, error)
}
type DeleteFileRequest struct {
@@ -229,6 +232,28 @@ func (c *remoteClient) CreateUploadSession(ctx context.Context, session *fs.Uplo
return nil
}
func (c *remoteClient) List(ctx context.Context, path string, recursive bool) ([]fs.PhysicalObject, error) {
resp, err := c.httpClient.Request(
http.MethodGet,
routes.SlaveFileListRoute(path, recursive),
nil,
request.WithContext(ctx),
request.WithLogger(c.l),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return nil, err
}
if resp.Code != 0 {
return nil, fmt.Errorf(resp.Error)
}
var objects []fs.PhysicalObject
resp.GobDecode(&objects)
return objects, nil
}
func (c *remoteClient) GetUploadURL(ctx context.Context, expires time.Time, sessionID string) (string, string, error) {
base, err := url.Parse(c.policy.Edges.Node.Server)
if err != nil {

109
pkg/filemanager/driver/remote/remote.go
View File

@@ -4,6 +4,10 @@ import (
"context"
"errors"
"fmt"
"net/url"
"os"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/auth"
@@ -15,10 +19,6 @@ import (
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/request"
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"net/url"
"os"
"path"
"time"
)
var (
@@ -54,108 +54,19 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
}, nil
}
//// List 列取文件
//func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
// var res []response.Object
//
// reqBody := serializer.ListRequest{
// Path: path,
// Recursive: recursive,
// }
// reqBodyEncoded, err := json.Marshal(reqBody)
// if err != nil {
// return res, err
// }
//
// // 发送列表请求
// bodyReader := strings.NewReader(string(reqBodyEncoded))
// signTTL := model.GetIntSetting("slave_api_timeout", 60)
// resp, err := handler.Client.Request(
// "POST",
// handler.getAPIUrl("list"),
// bodyReader,
// request.WithCredential(handler.AuthInstance, int64(signTTL)),
// request.WithMasterMeta(handler.settings.SiteBasic(ctx).ID, handler.settings.SiteURL(setting.UseFirstSiteUrl(ctx)).String()),
// ).CheckHTTPResponse(200).DecodeResponse()
// if err != nil {
// return res, err
// }
//
// // 处理列取结果
// if resp.Code != 0 {
// return res, errors.New(resp.Error)
// }
//
// if resStr, ok := resp.Data.(string); ok {
// err = json.Unmarshal([]byte(resStr), &res)
// if err != nil {
// return res, err
// }
// }
//
// return res, nil
//}
// getAPIUrl 获取接口请求地址
func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Edges.Node.Server)
// List 列取文件
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
res, err := handler.uploadClient.List(ctx, base, recursive)
if err != nil {
return ""
}
var controller *url.URL
switch scope {
case "delete":
controller, _ = url.Parse("/api/v3/slave/delete")
case "thumb":
controller, _ = url.Parse("/api/v3/slave/thumb")
case "list":
controller, _ = url.Parse("/api/v3/slave/list")
default:
controller = serverURL
return nil, err
}
for _, r := range routes {
controller.Path = path.Join(controller.Path, r)
}
return serverURL.ResolveReference(controller).String()
onProgress(len(res))
return res, nil
}
// Open 获取文件内容
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
//// 尝试获取速度限制
//speedLimit := 0
//if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
// speedLimit = user.Group.SpeedLimit
//}
//
//// 获取文件源地址
//downloadURL, err := handler.Source(ctx, path, nil, true, int64(speedLimit))
//if err != nil {
// return nil, err
//}
//
//// 获取文件数据流
//resp, err := handler.Client.Request(
// "GET",
// downloadURL,
// nil,
// request.WithContext(ctx),
// request.WithTimeout(time.Duration(0)),
// request.WithMasterMeta(handler.settings.SiteBasic(ctx).ID, handler.settings.SiteURL(ctx).String()),
//).CheckHTTPResponse(200).GetRSCloser()
//if err != nil {
// return nil, err
//}
//
//resp.SetFirstFakeChunk()
//
//// 尝试获取文件大小
//if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
// resp.SetContentLength(int64(file.Size))
//}
return nil, errors.New("not implemented")
}

158
pkg/filemanager/driver/s3/s3.go
View File

@@ -7,6 +7,9 @@ import (
"io"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -100,82 +103,85 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
return driver, nil
}
//// List 列出给定路径下的文件
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// // 初始化列目录参数
// base = strings.TrimPrefix(base, "/")
// if base != "" {
// base += "/"
// }
//
// opt := &s3.ListObjectsInput{
// Bucket: &handler.policy.BucketName,
// Prefix: &base,
// MaxKeys: aws.Int64(1000),
// }
//
// // 是否为递归列出
// if !recursive {
// opt.Delimiter = aws.String("/")
// }
//
// var (
// objects []*s3.Object
// commons []*s3.CommonPrefix
// )
//
// for {
// res, err := handler.svc.ListObjectsWithContext(ctx, opt)
// if err != nil {
// return nil, err
// }
// objects = append(objects, res.Contents...)
// commons = append(commons, res.CommonPrefixes...)
//
// // 如果本次未列取完，则继续使用marker获取结果
// if *res.IsTruncated {
// opt.Marker = res.NextMarker
// } else {
// break
// }
// }
//
// // 处理列取结果
// res := make([]response.Object, 0, len(objects)+len(commons))
//
// // 处理目录
// for _, object := range commons {
// rel, err := filepath.Rel(*opt.Prefix, *object.Prefix)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(*object.Prefix),
// RelativePath: filepath.ToSlash(rel),
// Size: 0,
// IsDir: true,
// LastModify: time.Now(),
// })
// }
// // 处理文件
// for _, object := range objects {
// rel, err := filepath.Rel(*opt.Prefix, *object.Key)
// if err != nil {
// continue
// }
// res = append(res, response.Object{
// Name: path.Base(*object.Key),
// Source: *object.Key,
// RelativePath: filepath.ToSlash(rel),
// Size: uint64(*object.Size),
// IsDir: false,
// LastModify: time.Now(),
// })
// }
//
// return res, nil
//
//}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
// 初始化列目录参数
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
opt := &s3.ListObjectsInput{
Bucket: &handler.policy.BucketName,
Prefix: &base,
MaxKeys: aws.Int64(1000),
}
// 是否为递归列出
if !recursive {
opt.Delimiter = aws.String("/")
}
var (
objects []*s3.Object
commons []*s3.CommonPrefix
)
for {
res, err := handler.svc.ListObjectsWithContext(ctx, opt)
if err != nil {
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完，则继续使用marker获取结果
if *res.IsTruncated {
opt.Marker = res.NextMarker
} else {
break
}
}
// 处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(*opt.Prefix, *object.Prefix)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(*object.Prefix),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
onProgress(len(commons))
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(*opt.Prefix, *object.Key)
if err != nil {
continue
}
res = append(res, fs.PhysicalObject{
Name: path.Base(*object.Key),
Source: *object.Key,
RelativePath: filepath.ToSlash(rel),
Size: int64(*object.Size),
IsDir: false,
LastModify: time.Now(),
})
}
onProgress(len(objects))
return res, nil
}
// Open 打开文件
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {

136
pkg/filemanager/driver/upyun/upyun.go
View File

@@ -10,6 +10,15 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/cloudreve/Cloudreve/v4/ent"
"github.com/cloudreve/Cloudreve/v4/inventory/types"
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
@@ -23,12 +32,6 @@ import (
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
"github.com/gin-gonic/gin"
"github.com/upyun/go-sdk/upyun"
"io"
"net/url"
"os"
"strconv"
"strings"
"time"
)
type (
@@ -78,66 +81,67 @@ func New(ctx context.Context, policy *ent.StoragePolicy, settings setting.Provid
return driver, nil
}
//func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// base = strings.TrimPrefix(base, "/")
//
// // 用于接受SDK返回对象的chan
// objChan := make(chan *upyun.FileInfo)
// objects := []*upyun.FileInfo{}
//
// // 列取配置
// listConf := &upyun.GetObjectsConfig{
// Path: "/" + base,
// ObjectsChan: objChan,
// MaxListTries: 1,
// }
// // 递归列取时不限制递归次数
// if recursive {
// listConf.MaxListLevel = -1
// }
//
// // 启动一个goroutine收集列取的对象信
// wg := &sync.WaitGroup{}
// wg.Add(1)
// go func(input chan *upyun.FileInfo, output *[]*upyun.FileInfo, wg *sync.WaitGroup) {
// defer wg.Done()
// for {
// file, ok := <-input
// if !ok {
// return
// }
// *output = append(*output, file)
// }
// }(objChan, &objects, wg)
//
// up := upyun.NewUpYun(&upyun.UpYunConfig{
// Bucket: handler.policy.BucketName,
// Operator: handler.policy.AccessKey,
// Password: handler.policy.SecretKey,
// })
//
// err := up.List(listConf)
// if err != nil {
// return nil, err
// }
//
// wg.Wait()
//
// // 汇总处理列取结果
// res := make([]response.Object, 0, len(objects))
// for _, object := range objects {
// res = append(res, response.Object{
// Name: path.Base(object.Name),
// RelativePath: object.Name,
// Source: path.Join(base, object.Name),
// Size: uint64(object.Size),
// IsDir: object.IsDir,
// LastModify: object.Time,
// })
// }
//
// return res, nil
//}
func (handler *Driver) List(ctx context.Context, base string, onProgress driver.ListProgressFunc, recursive bool) ([]fs.PhysicalObject, error) {
base = strings.TrimPrefix(base, "/")
// 用于接受SDK返回对象的chan
objChan := make(chan *upyun.FileInfo)
objects := []*upyun.FileInfo{}
// 列取配置
listConf := &upyun.GetObjectsConfig{
Path: "/" + base,
ObjectsChan: objChan,
MaxListTries: 1,
}
// 递归列取时不限制递归次数
if recursive {
listConf.MaxListLevel = -1
}
// 启动一个goroutine收集列取的对象信
wg := &sync.WaitGroup{}
wg.Add(1)
go func(input chan *upyun.FileInfo, output *[]*upyun.FileInfo, wg *sync.WaitGroup) {
defer wg.Done()
for {
file, ok := <-input
if !ok {
return
}
*output = append(*output, file)
onProgress(1)
}
}(objChan, &objects, wg)
up := upyun.NewUpYun(&upyun.UpYunConfig{
Bucket: handler.policy.BucketName,
Operator: handler.policy.AccessKey,
Password: handler.policy.SecretKey,
})
err := up.List(listConf)
if err != nil {
return nil, err
}
wg.Wait()
// 汇总处理列取结果
res := make([]fs.PhysicalObject, 0, len(objects))
for _, object := range objects {
res = append(res, fs.PhysicalObject{
Name: path.Base(object.Name),
RelativePath: object.Name,
Source: path.Join(base, object.Name),
Size: int64(object.Size),
IsDir: object.IsDir,
LastModify: object.Time,
})
}
return res, nil
}
func (handler *Driver) Open(ctx context.Context, path string) (*os.File, error) {
return nil, errors.New("not implemented")