Feat: support option for cache streamed chunk data into temp file for potential retry.

pkg/filesystem/chunk/chunk.go

@@ -7,29 +7,35 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 	"io"
+	"os"
 )
 
+const bufferTempPattern = "cdChunk.*.tmp"
+
 // ChunkProcessFunc callback function for processing a chunk
 type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
 
 // ChunkGroup manage groups of chunks
 type ChunkGroup struct {
-	file      fsctx.FileHeader
-	chunkSize uint64
-	backoff   backoff.Backoff
+	file              fsctx.FileHeader
+	chunkSize         uint64
+	backoff           backoff.Backoff
+	enableRetryBuffer bool
 
 	fileInfo     *fsctx.UploadTaskInfo
 	currentIndex int
 	chunkNum     uint64
+	bufferTemp   *os.File
 }
 
-func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff) *ChunkGroup {
+func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
 	c := &ChunkGroup{
-		file:         file,
-		chunkSize:    chunkSize,
-		backoff:      backoff,
-		fileInfo:     file.Info(),
-		currentIndex: -1,
+		file:              file,
+		chunkSize:         chunkSize,
+		backoff:           backoff,
+		fileInfo:          file.Info(),
+		currentIndex:      -1,
+		enableRetryBuffer: useBuffer,
 	}
 
 	if c.chunkSize == 0 {
@@ -44,13 +50,53 @@ func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Back
 	return c
 }
 
+// TempAvailable returns if current chunk temp file is available to be read
+func (c *ChunkGroup) TempAvailable() bool {
+	if c.bufferTemp != nil {
+		state, _ := c.bufferTemp.Stat()
+		return state != nil && state.Size() == c.Length()
+	}
+
+	return false
+}
+
 // Process a chunk with retry logic
 func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
-	err := processor(c, io.LimitReader(c.file, int64(c.chunkSize)))
+	reader := io.LimitReader(c.file, int64(c.chunkSize))
+
+	// If useBuffer is enabled, tee the reader to a temp file
+	if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
+		c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
+		reader = io.TeeReader(reader, c.bufferTemp)
+	}
+
+	if c.bufferTemp != nil {
+		defer func() {
+			if c.bufferTemp != nil {
+				c.bufferTemp.Close()
+				os.Remove(c.bufferTemp.Name())
+				c.bufferTemp = nil
+			}
+		}()
+
+		// if temp buffer file is available, use it
+		if c.TempAvailable() {
+			if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
+				return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
+			}
+
+			util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
+			reader = c.bufferTemp
+		}
+	}
+
+	err := processor(c, reader)
 	if err != nil {
-		if err != context.Canceled && c.file.Seekable() && c.backoff.Next() {
-			if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
-				return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err)
+		if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
+			if c.file.Seekable() {
+				if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
+					return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err)
+				}
 			}
 
 			util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)