LeonMMcoset/leonpan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Init V4 community edition (#2265)

Browse Source 
* Init V4 community edition

* Init V4 community edition
This commit is contained in:
AaronLiu
2025-04-20 17:31:25 +08:00
committed by GitHub
parent da4e44b77a
commit 21d158db07
597 changed files with 119415 additions and 41692 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pkg/cache/driver.go vendored
View File

@@ -2,102 +2,35 @@ package cache
import (
"encoding/gob"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
)
func init() {
gob.Register(map[string]itemWithTTL{})
}
// Store 缓存存储器
var Store Driver = NewMemoStore()
// Init 初始化缓存
func Init() {
if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode {
Store = NewRedisStore(
10,
conf.RedisConfig.Network,
conf.RedisConfig.Server,
conf.RedisConfig.User,
conf.RedisConfig.Password,
conf.RedisConfig.DB,
)
}
}
// Restore restores cache from given disk file
func Restore(persistFile string) {
if err := Store.Restore(persistFile); err != nil {
util.Log().Warning("Failed to restore cache from disk: %s", err)
}
}
func InitSlaveOverwrites() {
err := Store.Sets(conf.OptionOverwrite, "setting_")
if err != nil {
util.Log().Warning("Failed to overwrite database setting: %s", err)
}
}
// Driver 键值缓存存储容器
type Driver interface {
// 设置值，ttl为过期时间，单位为秒
Set(key string, value interface{}, ttl int) error
Set(key string, value any, ttl int) error
// 取值，并返回是否成功
Get(key string) (interface{}, bool)
Get(key string) (any, bool)
// 批量取值，返回成功取值的map即不存在的值
Gets(keys []string, prefix string) (map[string]interface{}, []string)
Gets(keys []string, prefix string) (map[string]any, []string)
// 批量设置值，所有的key都会加上prefix前缀
Sets(values map[string]interface{}, prefix string) error
Sets(values map[string]any, prefix string) error
// 删除值
Delete(keys []string, prefix string) error
// Delete values by [Prefix + key]. If no ket is presented, all keys with given prefix will be deleted.
Delete(prefix string, keys ...string) error
// Save in-memory cache to disk
Persist(path string) error
// Restore cache from disk
Restore(path string) error
}
// Set 设置缓存值
func Set(key string, value interface{}, ttl int) error {
return Store.Set(key, value, ttl)
}
// Get 获取缓存值
func Get(key string) (interface{}, bool) {
return Store.Get(key)
}
// Deletes 删除值
func Deletes(keys []string, prefix string) error {
return Store.Delete(keys, prefix)
}
// GetSettings 根据名称批量获取设置项缓存
func GetSettings(keys []string, prefix string) (map[string]string, []string) {
raw, miss := Store.Gets(keys, prefix)
res := make(map[string]string, len(raw))
for k, v := range raw {
res[k] = v.(string)
}
return res, miss
}
// SetSettings 批量设置站点设置缓存
func SetSettings(values map[string]string, prefix string) error {
var toBeSet = make(map[string]interface{}, len(values))
for key, value := range values {
toBeSet[key] = interface{}(value)
}
return Store.Sets(toBeSet, prefix)
// Remove all entries
DeleteAll() error
}

8
pkg/cache/driver_test.go vendored
View File

@@ -59,11 +59,3 @@ func TestInit(t *testing.T) {
Init()
})
}
func TestInitSlaveOverwrites(t *testing.T) {
asserts := assert.New(t)
asserts.NotPanics(func() {
InitSlaveOverwrites()
})
}

52
pkg/cache/memo.go vendored
View File

@@ -3,11 +3,12 @@ package cache
import (
"encoding/gob"
"fmt"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"github.com/cloudreve/Cloudreve/v4/pkg/util"
"os"
"strings"
"sync"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// MemoStore 内存存储驱动
@@ -35,7 +36,7 @@ func newItem(value interface{}, expires int) itemWithTTL {
}
// getValue 从itemWithTTL中取值
func getValue(item interface{}, ok bool) (interface{}, bool) {
func getValue(item any, ok bool) (any, bool) {
if !ok {
return nil, ok
}
@@ -55,7 +56,7 @@ func getValue(item interface{}, ok bool) (interface{}, bool) {
// GarbageCollect 回收已过期的缓存
func (store *MemoStore) GarbageCollect() {
store.Store.Range(func(key, value interface{}) bool {
store.Store.Range(func(key, value any) bool {
if item, ok := value.(itemWithTTL); ok {
if item.Expires > 0 && item.Expires < time.Now().Unix() {
util.Log().Debug("Cache %q is garbage collected.", key.(string))
@@ -67,25 +68,33 @@ func (store *MemoStore) GarbageCollect() {
}
// NewMemoStore 新建内存存储
func NewMemoStore() *MemoStore {
return &MemoStore{
func NewMemoStore(persistFile string, l logging.Logger) *MemoStore {
store := &MemoStore{
Store: &sync.Map{},
}
if persistFile != "" {
if err := store.Restore(persistFile); err != nil {
l.Warning("Failed to restore cache from disk: %s", err)
}
}
return store
}
// Set 存储值
func (store *MemoStore) Set(key string, value interface{}, ttl int) error {
func (store *MemoStore) Set(key string, value any, ttl int) error {
store.Store.Store(key, newItem(value, ttl))
return nil
}
// Get 取值
func (store *MemoStore) Get(key string) (interface{}, bool) {
func (store *MemoStore) Get(key string) (any, bool) {
return getValue(store.Store.Load(key))
}
// Gets 批量取值
func (store *MemoStore) Gets(keys []string, prefix string) (map[string]interface{}, []string) {
func (store *MemoStore) Gets(keys []string, prefix string) (map[string]any, []string) {
var res = make(map[string]interface{})
var notFound = make([]string, 0, len(keys))
@@ -101,7 +110,7 @@ func (store *MemoStore) Gets(keys []string, prefix string) (map[string]interface
}
// Sets 批量设置值
func (store *MemoStore) Sets(values map[string]interface{}, prefix string) error {
func (store *MemoStore) Sets(values map[string]any, prefix string) error {
for key, value := range values {
store.Store.Store(prefix+key, newItem(value, 0))
}
@@ -109,17 +118,27 @@ func (store *MemoStore) Sets(values map[string]interface{}, prefix string) error
}
// Delete 批量删除值
func (store *MemoStore) Delete(keys []string, prefix string) error {
func (store *MemoStore) Delete(prefix string, keys ...string) error {
for _, key := range keys {
store.Store.Delete(prefix + key)
}
// No key is presented, delete all entries with given prefix.
if len(keys) == 0 {
store.Store.Range(func(key, value any) bool {
if k, ok := key.(string); ok && strings.HasPrefix(k, prefix) {
store.Store.Delete(key)
}
return true
})
}
return nil
}
// Persist write memory store into cache
func (store *MemoStore) Persist(path string) error {
persisted := make(map[string]itemWithTTL)
store.Store.Range(func(key, value interface{}) bool {
store.Store.Range(func(key, value any) bool {
v, ok := store.Store.Load(key)
if _, ok := getValue(v, ok); ok {
persisted[key.(string)] = v.(itemWithTTL)
@@ -173,3 +192,12 @@ func (store *MemoStore) Restore(path string) error {
util.Log().Info("Restored %d items from %q into memory cache.", loaded, path)
return nil
}
func (store *MemoStore) DeleteAll() error {
store.Store.Range(func(key any, value any) bool {
store.Store.Delete(key)
return true
})
return nil
}

44
pkg/cache/memo_test.go vendored
View File

@@ -2,7 +2,6 @@ package cache
import (
"github.com/stretchr/testify/assert"
"path/filepath"
"testing"
"time"
)
@@ -146,46 +145,3 @@ func TestMemoStore_GarbageCollect(t *testing.T) {
_, ok := store.Get("test")
asserts.False(ok)
}
func TestMemoStore_PersistFailed(t *testing.T) {
a := assert.New(t)
store := NewMemoStore()
type testStruct struct{ v string }
store.Set("test", 1, 0)
store.Set("test2", testStruct{v: "test"}, 0)
err := store.Persist(filepath.Join(t.TempDir(), "TestMemoStore_PersistFailed"))
a.Error(err)
}
func TestMemoStore_PersistAndRestore(t *testing.T) {
a := assert.New(t)
store := NewMemoStore()
store.Set("test", 1, 0)
// already expired
store.Store.Store("test2", itemWithTTL{Value: "test", Expires: 1})
// expired after persist
store.Set("test3", 1, 1)
temp := filepath.Join(t.TempDir(), "TestMemoStore_PersistFailed")
// Persist
err := store.Persist(temp)
a.NoError(err)
a.FileExists(temp)
time.Sleep(2 * time.Second)
// Restore
store2 := NewMemoStore()
err = store2.Restore(temp)
a.NoError(err)
test, testOk := store2.Get("test")
a.EqualValues(1, test)
a.True(testOk)
test2, test2Ok := store2.Get("test2")
a.Nil(test2)
a.False(test2Ok)
test3, test3Ok := store2.Get("test3")
a.Nil(test3)
a.False(test3Ok)
a.NoFileExists(temp)
}

46
pkg/cache/redis.go vendored
View File

@@ -3,10 +3,10 @@ package cache
import (
"bytes"
"encoding/gob"
"github.com/cloudreve/Cloudreve/v4/pkg/logging"
"strconv"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gomodule/redigo/redis"
)
@@ -19,7 +19,7 @@ type item struct {
Value interface{}
}
func serializer(value interface{}) ([]byte, error) {
func serializer(value any) ([]byte, error) {
var buffer bytes.Buffer
enc := gob.NewEncoder(&buffer)
storeValue := item{
@@ -32,7 +32,7 @@ func serializer(value interface{}) ([]byte, error) {
return buffer.Bytes(), nil
}
func deserializer(value []byte) (interface{}, error) {
func deserializer(value []byte) (any, error) {
var res item
buffer := bytes.NewReader(value)
dec := gob.NewDecoder(buffer)
@@ -44,7 +44,7 @@ func deserializer(value []byte) (interface{}, error) {
}
// NewRedisStore 创建新的redis存储
func NewRedisStore(size int, network, address, user, password, database string) *RedisStore {
func NewRedisStore(l logging.Logger, size int, network, address, user, password, database string) *RedisStore {
return &RedisStore{
pool: &redis.Pool{
MaxIdle: size,
@@ -63,11 +63,11 @@ func NewRedisStore(size int, network, address, user, password, database string)
network,
address,
redis.DialDatabase(db),
redis.DialUsername(user),
redis.DialPassword(password),
redis.DialUsername(user),
)
if err != nil {
util.Log().Panic("Failed to create Redis connection: %s", err)
l.Panic("Failed to create Redis connection: %s", err)
}
return c, nil
},
@@ -76,7 +76,7 @@ func NewRedisStore(size int, network, address, user, password, database string)
}
// Set 存储值
func (store *RedisStore) Set(key string, value interface{}, ttl int) error {
func (store *RedisStore) Set(key string, value any, ttl int) error {
rc := store.pool.Get()
defer rc.Close()
@@ -103,7 +103,7 @@ func (store *RedisStore) Set(key string, value interface{}, ttl int) error {
}
// Get 取值
func (store *RedisStore) Get(key string) (interface{}, bool) {
func (store *RedisStore) Get(key string) (any, bool) {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
@@ -125,7 +125,7 @@ func (store *RedisStore) Get(key string) (interface{}, bool) {
}
// Gets 批量取值
func (store *RedisStore) Gets(keys []string, prefix string) (map[string]interface{}, []string) {
func (store *RedisStore) Gets(keys []string, prefix string) (map[string]any, []string) {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
@@ -142,7 +142,7 @@ func (store *RedisStore) Gets(keys []string, prefix string) (map[string]interfac
return nil, keys
}
var res = make(map[string]interface{})
var res = make(map[string]any)
var missed = make([]string, 0, len(keys))
for key, value := range v {
@@ -158,13 +158,13 @@ func (store *RedisStore) Gets(keys []string, prefix string) (map[string]interfac
}
// Sets 批量设置值
func (store *RedisStore) Sets(values map[string]interface{}, prefix string) error {
func (store *RedisStore) Sets(values map[string]any, prefix string) error {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
return rc.Err()
}
var setValues = make(map[string]interface{})
var setValues = make(map[string]any)
// 编码待设置值
for key, value := range values {
@@ -184,7 +184,7 @@ func (store *RedisStore) Sets(values map[string]interface{}, prefix string) erro
}
// Delete 批量删除给定的键
func (store *RedisStore) Delete(keys []string, prefix string) error {
func (store *RedisStore) Delete(prefix string, keys ...string) error {
rc := store.pool.Get()
defer rc.Close()
if rc.Err() != nil {
@@ -196,10 +196,24 @@ func (store *RedisStore) Delete(keys []string, prefix string) error {
keys[i] = prefix + keys[i]
}
_, err := rc.Do("DEL", redis.Args{}.AddFlat(keys)...)
if err != nil {
return err
// No key is presented, delete all keys with given prefix
if len(keys) == 0 {
// Fetch all key with given prefix
allPrefixKeys, err := redis.Strings(rc.Do("KEYS", prefix+"*"))
if err != nil {
return err
}
keys = allPrefixKeys
}
if len(keys) > 0 {
_, err := rc.Do("DEL", redis.Args{}.AddFlat(keys)...)
if err != nil {
return err
}
}
return nil
}

10
pkg/cache/redis_test.go vendored
View File

@@ -13,16 +13,16 @@ import (
func TestNewRedisStore(t *testing.T) {
asserts := assert.New(t)
store := NewRedisStore(10, "tcp", "", "", "", "0")
store := NewRedisStore(10, "tcp", "", "", "0")
asserts.NotNil(store)
asserts.Panics(func() {
store.pool.Dial()
})
conn, err := store.pool.Dial()
asserts.Nil(conn)
asserts.Error(err)
testConn := redigomock.NewConn()
cmd := testConn.Command("PING").Expect("PONG")
err := store.pool.TestOnBorrow(testConn, time.Now())
err = store.pool.TestOnBorrow(testConn, time.Now())
if testConn.Stats(cmd) != 1 {
fmt.Println("Command was not used")
return