Init V4 community edition (#2265)
* Init V4 community edition * Init V4 community edition
This commit is contained in:
AaronLiu
536
pkg/filemanager/fs/dbfs/navigator.go
Normal file
536
pkg/filemanager/fs/dbfs/navigator.go
Normal file
@@ -0,0 +1,536 @@
|
||||
package dbfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudreve/Cloudreve/v4/application/constants"
|
||||
"github.com/cloudreve/Cloudreve/v4/ent"
|
||||
"github.com/cloudreve/Cloudreve/v4/inventory"
|
||||
"github.com/cloudreve/Cloudreve/v4/inventory/types"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/boolset"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/cache"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v4/pkg/setting"
|
||||
"github.com/samber/lo"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFsNotInitialized = fmt.Errorf("fs not initialized")
|
||||
ErrPermissionDenied = serializer.NewError(serializer.CodeNoPermissionErr, "Permission denied", nil)
|
||||
|
||||
ErrShareIncorrectPassword = serializer.NewError(serializer.CodeIncorrectPassword, "Incorrect share password", nil)
|
||||
ErrFileCountLimitedReached = serializer.NewError(serializer.CodeFileCountLimitedReached, "Walked file count reached limit", nil)
|
||||
ErrSymbolicFolderFound = serializer.NewError(serializer.CodeNoPermissionErr, "Symbolic folder cannot be walked into", nil)
|
||||
ErrLoginRequired = serializer.NewError(serializer.CodeCheckLogin, "Login required", nil)
|
||||
|
||||
fullOrderByOption = []string{"name", "size", "updated_at", "created_at"}
|
||||
searchLimitedOrderByOption = []string{"created_at"}
|
||||
fullOrderDirectionOption = []string{"asc", "desc"}
|
||||
)
|
||||
|
||||
type (
|
||||
// Navigator is a navigator for database file system.
|
||||
Navigator interface {
|
||||
Recycle()
|
||||
// To returns the file by path. If given path is not exist, returns ErrFileNotFound and most-recent ancestor.
|
||||
To(ctx context.Context, path *fs.URI) (*File, error)
|
||||
// Children returns the children of the parent file.
|
||||
Children(ctx context.Context, parent *File, args *ListArgs) (*ListResult, error)
|
||||
// Capabilities returns the capabilities of the navigator.
|
||||
Capabilities(isSearching bool) *fs.NavigatorProps
|
||||
// Walk walks the file tree until limit is reached.
|
||||
Walk(ctx context.Context, levelFiles []*File, limit, depth int, f WalkFunc) error
|
||||
// PersistState tells navigator to persist the state of the navigator before recycle.
|
||||
PersistState(kv cache.Driver, key string)
|
||||
// RestoreState restores the state of the navigator.
|
||||
RestoreState(s State) error
|
||||
// FollowTx let the navigator inherit the transaction. Return a function to reset back to previous DB client.
|
||||
FollowTx(ctx context.Context) (func(), error)
|
||||
// ExecuteHook performs custom operations before or after certain actions.
|
||||
ExecuteHook(ctx context.Context, hookType fs.HookType, file *File) error
|
||||
}
|
||||
|
||||
State interface{}
|
||||
|
||||
NavigatorCapability int
|
||||
ListArgs struct {
|
||||
Page *inventory.PaginationArgs
|
||||
Search *inventory.SearchFileParameters
|
||||
SharedWithMe bool
|
||||
StreamCallback func([]*File)
|
||||
}
|
||||
// ListResult is the result of a list operation.
|
||||
ListResult struct {
|
||||
Files []*File
|
||||
MixedType bool
|
||||
Pagination *inventory.PaginationResults
|
||||
RecursionLimitReached bool
|
||||
SingleFileView bool
|
||||
}
|
||||
WalkFunc func([]*File, int) error
|
||||
)
|
||||
|
||||
const (
|
||||
NavigatorCapabilityCreateFile NavigatorCapability = iota
|
||||
NavigatorCapabilityRenameFile
|
||||
NavigatorCapability_CommunityPlacehodler1
|
||||
NavigatorCapability_CommunityPlacehodler2
|
||||
NavigatorCapability_CommunityPlacehodler3
|
||||
NavigatorCapability_CommunityPlacehodler4
|
||||
NavigatorCapabilityUploadFile
|
||||
NavigatorCapabilityDownloadFile
|
||||
NavigatorCapabilityUpdateMetadata
|
||||
NavigatorCapabilityListChildren
|
||||
NavigatorCapabilityGenerateThumb
|
||||
NavigatorCapability_CommunityPlacehodler5
|
||||
NavigatorCapability_CommunityPlacehodler6
|
||||
NavigatorCapability_CommunityPlacehodler7
|
||||
NavigatorCapabilityDeleteFile
|
||||
NavigatorCapabilityLockFile
|
||||
NavigatorCapabilitySoftDelete
|
||||
NavigatorCapabilityRestore
|
||||
NavigatorCapabilityShare
|
||||
NavigatorCapabilityInfo
|
||||
NavigatorCapabilityVersionControl
|
||||
NavigatorCapability_CommunityPlacehodler8
|
||||
NavigatorCapability_CommunityPlacehodler9
|
||||
NavigatorCapabilityEnterFolder
|
||||
|
||||
searchTokenSeparator = "|"
|
||||
)
|
||||
|
||||
func init() {
|
||||
boolset.Sets(map[NavigatorCapability]bool{
|
||||
NavigatorCapabilityCreateFile: true,
|
||||
NavigatorCapabilityRenameFile: true,
|
||||
NavigatorCapabilityUploadFile: true,
|
||||
NavigatorCapabilityDownloadFile: true,
|
||||
NavigatorCapabilityUpdateMetadata: true,
|
||||
NavigatorCapabilityListChildren: true,
|
||||
NavigatorCapabilityGenerateThumb: true,
|
||||
NavigatorCapabilityDeleteFile: true,
|
||||
NavigatorCapabilityLockFile: true,
|
||||
NavigatorCapabilitySoftDelete: true,
|
||||
NavigatorCapabilityShare: true,
|
||||
NavigatorCapabilityInfo: true,
|
||||
NavigatorCapabilityVersionControl: true,
|
||||
NavigatorCapabilityEnterFolder: true,
|
||||
}, myNavigatorCapability)
|
||||
boolset.Sets(map[NavigatorCapability]bool{
|
||||
NavigatorCapabilityDownloadFile: true,
|
||||
NavigatorCapabilityListChildren: true,
|
||||
NavigatorCapabilityGenerateThumb: true,
|
||||
NavigatorCapabilityLockFile: true,
|
||||
NavigatorCapabilityInfo: true,
|
||||
NavigatorCapabilityVersionControl: true,
|
||||
NavigatorCapabilityEnterFolder: true,
|
||||
}, shareNavigatorCapability)
|
||||
boolset.Sets(map[NavigatorCapability]bool{
|
||||
NavigatorCapabilityListChildren: true,
|
||||
NavigatorCapabilityDeleteFile: true,
|
||||
NavigatorCapabilityLockFile: true,
|
||||
NavigatorCapabilityRestore: true,
|
||||
NavigatorCapabilityInfo: true,
|
||||
}, trashNavigatorCapability)
|
||||
boolset.Sets(map[NavigatorCapability]bool{
|
||||
NavigatorCapabilityListChildren: true,
|
||||
NavigatorCapabilityDownloadFile: true,
|
||||
NavigatorCapabilityEnterFolder: true,
|
||||
}, sharedWithMeNavigatorCapability)
|
||||
}
|
||||
|
||||
// ==================== Base Navigator ====================
|
||||
type (
|
||||
fileFilter func(ctx context.Context, f *File) (*File, bool)
|
||||
baseNavigator struct {
|
||||
fileClient inventory.FileClient
|
||||
listFilter fileFilter
|
||||
user *ent.User
|
||||
hasher hashid.Encoder
|
||||
config *setting.DBFS
|
||||
}
|
||||
)
|
||||
|
||||
var defaultFilter = func(ctx context.Context, f *File) (*File, bool) { return f, true }
|
||||
|
||||
func newBaseNavigator(fileClient inventory.FileClient, filterFunc fileFilter, user *ent.User,
|
||||
hasher hashid.Encoder, config *setting.DBFS) *baseNavigator {
|
||||
return &baseNavigator{
|
||||
fileClient: fileClient,
|
||||
listFilter: filterFunc,
|
||||
user: user,
|
||||
hasher: hasher,
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *baseNavigator) walkNext(ctx context.Context, root *File, next string, isLeaf bool) (*File, error) {
|
||||
var model *ent.File
|
||||
if root != nil {
|
||||
model = root.Model
|
||||
if root.IsSymbolic() {
|
||||
return nil, ErrSymbolicFolderFound
|
||||
}
|
||||
|
||||
root.mu.Lock()
|
||||
if child, ok := root.Children[next]; ok && !isLeaf {
|
||||
root.mu.Unlock()
|
||||
return child, nil
|
||||
}
|
||||
root.mu.Unlock()
|
||||
}
|
||||
|
||||
child, err := b.fileClient.GetChildFile(ctx, model, b.user.ID, next, isLeaf)
|
||||
if err != nil {
|
||||
if ent.IsNotFound(err) {
|
||||
return nil, fs.ErrPathNotExist.WithError(err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("faield to get child %q: %w", next, err)
|
||||
}
|
||||
|
||||
return newFile(root, child), nil
|
||||
}
|
||||
|
||||
func (b *baseNavigator) walkUp(ctx context.Context, child *File) (*File, error) {
|
||||
parent, err := b.fileClient.GetParentFile(ctx, child.Model, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("faield to get Parent for %q: %w", child.Name(), err)
|
||||
}
|
||||
|
||||
return newParentFile(parent, child), nil
|
||||
}
|
||||
|
||||
func (b *baseNavigator) children(ctx context.Context, parent *File, args *ListArgs) (*ListResult, error) {
|
||||
var model *ent.File
|
||||
if parent != nil {
|
||||
model = parent.Model
|
||||
if parent.Model.Type != int(types.FileTypeFolder) {
|
||||
return nil, fs.ErrPathNotExist
|
||||
}
|
||||
|
||||
if parent.IsSymbolic() {
|
||||
return nil, ErrSymbolicFolderFound
|
||||
}
|
||||
|
||||
parent.Path[pathIndexUser] = parent.Uri(false)
|
||||
}
|
||||
|
||||
if args.Search != nil {
|
||||
return b.search(ctx, parent, args)
|
||||
}
|
||||
|
||||
children, err := b.fileClient.GetChildFiles(ctx, &inventory.ListFileParameters{
|
||||
PaginationArgs: args.Page,
|
||||
SharedWithMe: args.SharedWithMe,
|
||||
}, b.user.ID, model)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get children: %w", err)
|
||||
}
|
||||
|
||||
return &ListResult{
|
||||
Files: lo.FilterMap(children.Files, func(model *ent.File, index int) (*File, bool) {
|
||||
f := newFile(parent, model)
|
||||
return b.listFilter(ctx, f)
|
||||
}),
|
||||
MixedType: children.MixedType,
|
||||
Pagination: children.PaginationResults,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *baseNavigator) walk(ctx context.Context, levelFiles []*File, limit, depth int, f WalkFunc) error {
|
||||
walked := 0
|
||||
if len(levelFiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
owner := levelFiles[0].Owner()
|
||||
|
||||
level := 0
|
||||
for walked <= limit && depth >= 0 {
|
||||
if len(levelFiles) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
stop := false
|
||||
depth--
|
||||
if len(levelFiles) > limit-walked {
|
||||
levelFiles = levelFiles[:limit-walked]
|
||||
stop = true
|
||||
}
|
||||
if err := f(levelFiles, level); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if stop {
|
||||
return ErrFileCountLimitedReached
|
||||
}
|
||||
|
||||
walked += len(levelFiles)
|
||||
folders := lo.Filter(levelFiles, func(f *File, index int) bool {
|
||||
return f.Model.Type == int(types.FileTypeFolder) && !f.IsSymbolic()
|
||||
})
|
||||
|
||||
if walked >= limit || len(folders) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
levelFiles = levelFiles[:0]
|
||||
leftCredit := limit - walked
|
||||
parents := lo.SliceToMap(folders, func(file *File) (int, *File) {
|
||||
return file.Model.ID, file
|
||||
})
|
||||
for leftCredit > 0 {
|
||||
token := ""
|
||||
res, err := b.fileClient.GetChildFiles(ctx,
|
||||
&inventory.ListFileParameters{
|
||||
PaginationArgs: &inventory.PaginationArgs{
|
||||
UseCursorPagination: true,
|
||||
PageToken: token,
|
||||
PageSize: leftCredit,
|
||||
},
|
||||
MixedType: true,
|
||||
},
|
||||
owner.ID,
|
||||
lo.Map(folders, func(item *File, index int) *ent.File {
|
||||
return item.Model
|
||||
})...)
|
||||
if err != nil {
|
||||
return serializer.NewError(serializer.CodeDBError, "Failed to list children", err)
|
||||
}
|
||||
|
||||
leftCredit -= len(res.Files)
|
||||
|
||||
levelFiles = append(levelFiles, lo.Map(res.Files, func(model *ent.File, index int) *File {
|
||||
p := parents[model.FileChildren]
|
||||
return newFile(p, model)
|
||||
})...)
|
||||
|
||||
// All files listed
|
||||
if res.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
token = res.NextPageToken
|
||||
}
|
||||
level++
|
||||
}
|
||||
|
||||
if walked >= limit {
|
||||
return ErrFileCountLimitedReached
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *baseNavigator) search(ctx context.Context, parent *File, args *ListArgs) (*ListResult, error) {
|
||||
if parent == nil {
|
||||
// Performs mega search for all files in trash fs.
|
||||
children, err := b.fileClient.GetChildFiles(ctx, &inventory.ListFileParameters{
|
||||
PaginationArgs: args.Page,
|
||||
MixedType: true,
|
||||
Search: args.Search,
|
||||
SharedWithMe: args.SharedWithMe,
|
||||
}, b.user.ID, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get children: %w", err)
|
||||
}
|
||||
|
||||
return &ListResult{
|
||||
Files: lo.FilterMap(children.Files, func(model *ent.File, index int) (*File, bool) {
|
||||
f := newFile(parent, model)
|
||||
return b.listFilter(ctx, f)
|
||||
}),
|
||||
MixedType: children.MixedType,
|
||||
Pagination: children.PaginationResults,
|
||||
}, nil
|
||||
}
|
||||
// Performs recursive search for all files under the given folder.
|
||||
walkedFolder := 1
|
||||
parents := []map[int]*File{{parent.Model.ID: parent}}
|
||||
startLevel, innerPageToken, err := parseSearchPageToken(args.Page.PageToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args.Page.PageToken = innerPageToken
|
||||
|
||||
stepLevel := func(level int) (bool, error) {
|
||||
token := ""
|
||||
// We don't need metadata in level search.
|
||||
listCtx := context.WithValue(ctx, inventory.LoadFilePublicMetadata{}, nil)
|
||||
for walkedFolder <= b.config.MaxRecursiveSearchedFolder {
|
||||
// TODO: chunk parents into 30000 per group
|
||||
res, err := b.fileClient.GetChildFiles(listCtx,
|
||||
&inventory.ListFileParameters{
|
||||
PaginationArgs: &inventory.PaginationArgs{
|
||||
UseCursorPagination: true,
|
||||
PageToken: token,
|
||||
},
|
||||
FolderOnly: true,
|
||||
},
|
||||
parent.Model.OwnerID,
|
||||
lo.MapToSlice(parents[level], func(k int, f *File) *ent.File {
|
||||
return f.Model
|
||||
})...)
|
||||
if err != nil {
|
||||
return false, serializer.NewError(serializer.CodeDBError, "Failed to list children", err)
|
||||
}
|
||||
|
||||
parents = append(parents, lo.SliceToMap(
|
||||
lo.FilterMap(res.Files, func(model *ent.File, index int) (*File, bool) {
|
||||
p := parents[level][model.FileChildren]
|
||||
f := newFile(p, model)
|
||||
f.Path[pathIndexUser] = p.Uri(false).Join(model.Name)
|
||||
return f, true
|
||||
}),
|
||||
func(f *File) (int, *File) {
|
||||
return f.Model.ID, f
|
||||
}))
|
||||
|
||||
walkedFolder += len(parents[level+1])
|
||||
if res.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
|
||||
token = res.NextPageToken
|
||||
}
|
||||
|
||||
if len(parents) <= level+1 || len(parents[level+1]) == 0 {
|
||||
// All possible folders is searched
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// We need to walk from root folder to get the correct level.
|
||||
for level := 0; level < startLevel; level++ {
|
||||
stop, err := stepLevel(level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if stop {
|
||||
return &ListResult{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Search files starting from current level
|
||||
res := make([]*File, 0, args.Page.PageSize)
|
||||
args.Page.UseCursorPagination = true
|
||||
originalPageSize := args.Page.PageSize
|
||||
stop := false
|
||||
for len(res) < originalPageSize && walkedFolder <= b.config.MaxRecursiveSearchedFolder {
|
||||
// Only requires minimum number of files
|
||||
args.Page.PageSize = min(originalPageSize, originalPageSize-len(res))
|
||||
searchRes, err := b.fileClient.GetChildFiles(ctx,
|
||||
&inventory.ListFileParameters{
|
||||
PaginationArgs: args.Page,
|
||||
MixedType: true,
|
||||
Search: args.Search,
|
||||
},
|
||||
parent.Model.OwnerID,
|
||||
lo.MapToSlice(parents[startLevel], func(k int, f *File) *ent.File {
|
||||
return f.Model
|
||||
})...)
|
||||
|
||||
if err != nil {
|
||||
return nil, serializer.NewError(serializer.CodeDBError, "Failed to search files", err)
|
||||
}
|
||||
|
||||
newRes := lo.FilterMap(searchRes.Files, func(model *ent.File, index int) (*File, bool) {
|
||||
p := parents[startLevel][model.FileChildren]
|
||||
f := newFile(p, model)
|
||||
f.Path[pathIndexUser] = p.Uri(false).Join(model.Name)
|
||||
return b.listFilter(ctx, f)
|
||||
})
|
||||
res = append(res, newRes...)
|
||||
if args.StreamCallback != nil {
|
||||
args.StreamCallback(newRes)
|
||||
}
|
||||
|
||||
args.Page.PageToken = searchRes.NextPageToken
|
||||
// If no more results under current level, move to next level
|
||||
if args.Page.PageToken == "" {
|
||||
if len(res) == originalPageSize {
|
||||
// Current page is full, no need to search more
|
||||
startLevel++
|
||||
break
|
||||
}
|
||||
|
||||
finished, err := stepLevel(startLevel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if finished {
|
||||
stop = true
|
||||
// No more folders under next level, all result is presented
|
||||
break
|
||||
}
|
||||
|
||||
startLevel++
|
||||
}
|
||||
}
|
||||
|
||||
if args.StreamCallback != nil {
|
||||
// Clear res if it's streamed
|
||||
res = res[:0]
|
||||
}
|
||||
|
||||
searchRes := &ListResult{
|
||||
Files: res,
|
||||
MixedType: true,
|
||||
Pagination: &inventory.PaginationResults{IsCursor: true},
|
||||
RecursionLimitReached: walkedFolder > b.config.MaxRecursiveSearchedFolder,
|
||||
}
|
||||
|
||||
if walkedFolder <= b.config.MaxRecursiveSearchedFolder && !stop {
|
||||
searchRes.Pagination.NextPageToken = fmt.Sprintf("%d%s%s", startLevel, searchTokenSeparator, args.Page.PageToken)
|
||||
}
|
||||
|
||||
return searchRes, nil
|
||||
}
|
||||
|
||||
func parseSearchPageToken(token string) (int, string, error) {
|
||||
if token == "" {
|
||||
return 0, "", nil
|
||||
}
|
||||
|
||||
tokens := strings.Split(token, searchTokenSeparator)
|
||||
if len(tokens) != 2 {
|
||||
return 0, "", fmt.Errorf("invalid page token")
|
||||
}
|
||||
|
||||
level, err := strconv.Atoi(tokens[0])
|
||||
if err != nil || level < 0 {
|
||||
return 0, "", fmt.Errorf("invalid page token level")
|
||||
}
|
||||
|
||||
return level, tokens[1], nil
|
||||
}
|
||||
|
||||
func newMyUri() *fs.URI {
|
||||
res, _ := fs.NewUriFromString(constants.CloudreveScheme + "://" + string(constants.FileSystemMy))
|
||||
return res
|
||||
}
|
||||
|
||||
func newMyIDUri(uid string) *fs.URI {
|
||||
res, _ := fs.NewUriFromString(fmt.Sprintf("%s://%s@%s", constants.CloudreveScheme, uid, constants.FileSystemMy))
|
||||
return res
|
||||
}
|
||||
|
||||
func newTrashUri(name string) *fs.URI {
|
||||
res, _ := fs.NewUriFromString(fmt.Sprintf("%s://%s", constants.CloudreveScheme, constants.FileSystemTrash))
|
||||
return res.Join(name)
|
||||
}
|
||||
|
||||
func newSharedWithMeUri(id string) *fs.URI {
|
||||
res, _ := fs.NewUriFromString(fmt.Sprintf("%s://%s", constants.CloudreveScheme, constants.FileSystemSharedWithMe))
|
||||
return res.Join(id)
|
||||
}
|
||||
Reference in New Issue
Block a user