feat(workflow): import files from external storage

pkg/filemanager/manager/entity.go

@@ -11,11 +11,13 @@ import (
 	"github.com/cloudreve/Cloudreve/v4/ent/user"
 	"github.com/cloudreve/Cloudreve/v4/inventory/types"
 	"github.com/cloudreve/Cloudreve/v4/pkg/cluster/routes"
+	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/driver"
 	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs"
 	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/fs/dbfs"
 	"github.com/cloudreve/Cloudreve/v4/pkg/filemanager/manager/entitysource"
 	"github.com/cloudreve/Cloudreve/v4/pkg/hashid"
 	"github.com/cloudreve/Cloudreve/v4/pkg/serializer"
+	"github.com/gofrs/uuid"
 	"github.com/samber/lo"
 )
 
@@ -41,6 +43,10 @@ type (
 		ExtractAndSaveMediaMeta(ctx context.Context, uri *fs.URI, entityID int) error
 		// RecycleEntities recycles a group of entities
 		RecycleEntities(ctx context.Context, force bool, entityIDs ...int) error
+		// ListPhysical lists physical files in a path
+		ListPhysical(ctx context.Context, path string, policyID int, recursive bool, progress driver.ListProgressFunc) ([]fs.PhysicalObject, error)
+		// ImportPhysical imports a physical file to a Cloudreve file
+		ImportPhysical(ctx context.Context, dst *fs.URI, policyId int, src fs.PhysicalObject, completeHook bool) error
 	}
 	DirectLink struct {
 		File fs.File
@@ -369,6 +375,51 @@ func (l *manager) DeleteVersion(ctx context.Context, path *fs.URI, version int)
 	return l.fs.VersionControl(ctx, path, version, true)
 }
 
+func (l *manager) ListPhysical(ctx context.Context, path string, policyID int, recursive bool, progress driver.ListProgressFunc) ([]fs.PhysicalObject, error) {
+	policy, err := l.dep.StoragePolicyClient().GetPolicyByID(ctx, policyID)
+	if err != nil {
+		return nil, err
+	}
+
+	driver, err := l.GetStorageDriver(ctx, policy)
+	if err != nil {
+		return nil, err
+	}
+
+	return driver.List(ctx, path, progress, recursive)
+}
+
+func (l *manager) ImportPhysical(ctx context.Context, dst *fs.URI, policyId int, src fs.PhysicalObject, completeHook bool) error {
+	targetUri := dst.Join(src.RelativePath)
+	req := &fs.UploadRequest{
+		Props: &fs.UploadProps{
+			Uri:                    targetUri,
+			UploadSessionID:        uuid.Must(uuid.NewV4()).String(),
+			Size:                   src.Size,
+			PreferredStoragePolicy: policyId,
+			SavePath:               src.Source,
+			LastModified:           &src.LastModify,
+		},
+		ImportFrom: &src,
+	}
+
+	// Prepare for upload
+	uploadSession, err := l.fs.PrepareUpload(ctx, req)
+	if err != nil {
+		return fmt.Errorf("faield to prepare uplaod: %w", err)
+	}
+	if completeHook {
+		d, err := l.GetStorageDriver(ctx, l.CastStoragePolicyOnSlave(ctx, uploadSession.Policy))
+		if err != nil {
+			return err
+		}
+
+		l.onNewEntityUploaded(ctx, uploadSession, d)
+	}
+
+	return nil
+}
+
 func entityUrlCacheKey(id int, speed int64, displayName string, download bool, siteUrl string) string {
 	hash := sha1.New()
 	hash.Write([]byte(fmt.Sprintf("%d_%d_%s_%t_%s", id,

pkg/filemanager/manager/recycle.go

@@ -219,9 +219,13 @@ func (m *manager) RecycleEntities(ctx context.Context, force bool, entityIDs ...
 				mapSrcToId[entity.Source()] = entity.ID()
 			}
 
-			res, err := d.Delete(ctx, lo.Map(chunk, func(entity fs.Entity, index int) string {
+			toBeDeletedSrc := lo.Map(lo.Filter(chunk, func(item fs.Entity, index int) bool {
+				// Only delete entities that are not marked as "unlink only"
+				return item.Model().RecycleOptions == nil || !item.Model().RecycleOptions.UnlinkOnly
+			}), func(entity fs.Entity, index int) string {
 				return entity.Source()
-			})...)
+			})
+			res, err := d.Delete(ctx, toBeDeletedSrc...)
 			if err != nil {
 				for _, src := range res {
 					ae.Add(strconv.Itoa(mapSrcToId[src]), err)

pkg/filemanager/manager/thumbnail.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/cloudreve/Cloudreve/v4/pkg/thumb"
 	"os"
 	"runtime"
 	"time"
@@ -270,6 +271,11 @@ func (m *GenerateThumbTask) Do(ctx context.Context) (task.Status, error) {
 
 	res, err := m.m.generateThumb(ctx, m.uri, m.ext, m.es)
 	if err != nil {
+		if errors.Is(err, thumb.ErrNotAvailable) {
+			m.sig <- &generateRes{nil, err}
+			return task.StatusCompleted, nil
+		}
+
 		return task.StatusError, err
 	}
 

pkg/filemanager/manager/upload.go

@@ -339,7 +339,7 @@ func (m *manager) OnUploadFailed(ctx context.Context, session *fs.UploadSession)
 			if err := m.Delete(ctx, []*fs.URI{session.Props.Uri}, fs.WithSysSkipSoftDelete(true)); err != nil {
 				m.l.Warning("OnUploadFailed hook failed to delete file: %s", err)
 			}
-		} else {
+		} else if !session.Importing {
 			if err := m.fs.VersionControl(ctx, session.Props.Uri, session.EntityID, true); err != nil {
 				m.l.Warning("OnUploadFailed hook failed to version control: %s", err)
 			}