Feat: aria2 download and transfer in slave node (#1040)
* Feat: retrieve nodes from data table * Feat: master node ping slave node in REST API * Feat: master send scheduled ping request * Feat: inactive nodes recover loop * Modify: remove database operations from aria2 RPC caller implementation * Feat: init aria2 client in master node * Feat: Round Robin load balancer * Feat: create and monitor aria2 task in master node * Feat: salve receive and handle heartbeat * Fix: Node ID will be 0 in download record generated in older version * Feat: sign request headers with all `X-` prefix * Feat: API call to slave node will carry meta data in headers * Feat: call slave aria2 rpc method from master * Feat: get slave aria2 task status Feat: encode slave response data using gob * Feat: aria2 callback to master node / cancel or select task to slave node * Fix: use dummy aria2 client when caller initialize failed in master node * Feat: slave aria2 status event callback / salve RPC auth * Feat: prototype for slave driven filesystem * Feat: retry for init aria2 client in master node * Feat: init request client with global options * Feat: slave receive async task from master * Fix: competition write in request header * Refactor: dependency initialize order * Feat: generic message queue implementation * Feat: message queue implementation * Feat: master waiting slave transfer result * Feat: slave transfer file in stateless policy * Feat: slave transfer file in slave policy * Feat: slave transfer file in local policy * Feat: slave transfer file in OneDrive policy * Fix: failed to initialize update checker http client * Feat: list slave nodes for dashboard * Feat: test aria2 rpc connection in slave * Feat: add and save node * Feat: add and delete node in node pool * Fix: temp file cannot be removed when aria2 task fails * Fix: delete node in admin panel * Feat: edit node and get node info * Modify: delete unused settings
This commit is contained in:
AaronLiu
110
pkg/request/options.go
Normal file
110
pkg/request/options.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package request
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Option 发送请求的额外设置
|
||||
type Option interface {
|
||||
apply(*options)
|
||||
}
|
||||
|
||||
type options struct {
|
||||
timeout time.Duration
|
||||
header http.Header
|
||||
sign auth.Auth
|
||||
signTTL int64
|
||||
ctx context.Context
|
||||
contentLength int64
|
||||
masterMeta bool
|
||||
endpoint *url.URL
|
||||
slaveNodeID string
|
||||
}
|
||||
|
||||
type optionFunc func(*options)
|
||||
|
||||
func (f optionFunc) apply(o *options) {
|
||||
f(o)
|
||||
}
|
||||
|
||||
func newDefaultOption() *options {
|
||||
return &options{
|
||||
header: http.Header{},
|
||||
timeout: time.Duration(30) * time.Second,
|
||||
contentLength: -1,
|
||||
}
|
||||
}
|
||||
|
||||
// WithTimeout 设置请求超时
|
||||
func WithTimeout(t time.Duration) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.timeout = t
|
||||
})
|
||||
}
|
||||
|
||||
// WithContext 设置请求上下文
|
||||
func WithContext(c context.Context) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.ctx = c
|
||||
})
|
||||
}
|
||||
|
||||
// WithCredential 对请求进行签名
|
||||
func WithCredential(instance auth.Auth, ttl int64) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.sign = instance
|
||||
o.signTTL = ttl
|
||||
})
|
||||
}
|
||||
|
||||
// WithHeader 设置请求Header
|
||||
func WithHeader(header http.Header) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
for k, v := range header {
|
||||
o.header[k] = v
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// WithoutHeader 设置清除请求Header
|
||||
func WithoutHeader(header []string) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
for _, v := range header {
|
||||
delete(o.header, v)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
// WithContentLength 设置请求大小
|
||||
func WithContentLength(s int64) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.contentLength = s
|
||||
})
|
||||
}
|
||||
|
||||
// WithMasterMeta 请求时携带主机信息
|
||||
func WithMasterMeta() Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.masterMeta = true
|
||||
})
|
||||
}
|
||||
|
||||
// WithSlaveMeta 请求时携带从机信息
|
||||
func WithSlaveMeta(s string) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.slaveNodeID = s
|
||||
})
|
||||
}
|
||||
|
||||
// Endpoint 使用同一的请求Endpoint
|
||||
func WithEndpoint(endpoint string) Option {
|
||||
endpointURL, _ := url.Parse(endpoint)
|
||||
return optionFunc(func(o *options) {
|
||||
o.endpoint = endpointURL
|
||||
})
|
||||
}
|
||||
@@ -1,23 +1,25 @@
|
||||
package request
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"time"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
)
|
||||
|
||||
// GeneralClient 通用 HTTP Client
|
||||
var GeneralClient Client = HTTPClient{}
|
||||
var GeneralClient Client = NewClient()
|
||||
|
||||
// Response 请求的响应或错误信息
|
||||
type Response struct {
|
||||
@@ -32,90 +34,30 @@ type Client interface {
|
||||
|
||||
// HTTPClient 实现 Client 接口
|
||||
type HTTPClient struct {
|
||||
mu sync.Mutex
|
||||
options *options
|
||||
}
|
||||
|
||||
// Option 发送请求的额外设置
|
||||
type Option interface {
|
||||
apply(*options)
|
||||
}
|
||||
|
||||
type options struct {
|
||||
timeout time.Duration
|
||||
header http.Header
|
||||
sign auth.Auth
|
||||
signTTL int64
|
||||
ctx context.Context
|
||||
contentLength int64
|
||||
}
|
||||
|
||||
type optionFunc func(*options)
|
||||
|
||||
func (f optionFunc) apply(o *options) {
|
||||
f(o)
|
||||
}
|
||||
|
||||
func newDefaultOption() *options {
|
||||
return &options{
|
||||
header: http.Header{},
|
||||
timeout: time.Duration(30) * time.Second,
|
||||
contentLength: -1,
|
||||
func NewClient(opts ...Option) Client {
|
||||
client := &HTTPClient{
|
||||
options: newDefaultOption(),
|
||||
}
|
||||
}
|
||||
|
||||
// WithTimeout 设置请求超时
|
||||
func WithTimeout(t time.Duration) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.timeout = t
|
||||
})
|
||||
}
|
||||
for _, o := range opts {
|
||||
o.apply(client.options)
|
||||
}
|
||||
|
||||
// WithContext 设置请求上下文
|
||||
func WithContext(c context.Context) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.ctx = c
|
||||
})
|
||||
}
|
||||
|
||||
// WithCredential 对请求进行签名
|
||||
func WithCredential(instance auth.Auth, ttl int64) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.sign = instance
|
||||
o.signTTL = ttl
|
||||
})
|
||||
}
|
||||
|
||||
// WithHeader 设置请求Header
|
||||
func WithHeader(header http.Header) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
for k, v := range header {
|
||||
o.header[k] = v
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// WithoutHeader 设置清除请求Header
|
||||
func WithoutHeader(header []string) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
for _, v := range header {
|
||||
delete(o.header, v)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
// WithContentLength 设置请求大小
|
||||
func WithContentLength(s int64) Option {
|
||||
return optionFunc(func(o *options) {
|
||||
o.contentLength = s
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
// Request 发送HTTP请求
|
||||
func (c HTTPClient) Request(method, target string, body io.Reader, opts ...Option) *Response {
|
||||
// 应用额外设置
|
||||
options := newDefaultOption()
|
||||
c.mu.Lock()
|
||||
options := *c.options
|
||||
c.mu.Unlock()
|
||||
for _, o := range opts {
|
||||
o.apply(options)
|
||||
o.apply(&options)
|
||||
}
|
||||
|
||||
// 创建请求客户端
|
||||
@@ -126,6 +68,13 @@ func (c HTTPClient) Request(method, target string, body io.Reader, opts ...Optio
|
||||
body = nil
|
||||
}
|
||||
|
||||
// 确定请求URL
|
||||
if options.endpoint != nil {
|
||||
targetURL := *options.endpoint
|
||||
targetURL.Path = path.Join(targetURL.Path, target)
|
||||
target = targetURL.String()
|
||||
}
|
||||
|
||||
// 创建请求
|
||||
var (
|
||||
req *http.Request
|
||||
@@ -141,14 +90,36 @@ func (c HTTPClient) Request(method, target string, body io.Reader, opts ...Optio
|
||||
}
|
||||
|
||||
// 添加请求相关设置
|
||||
req.Header = options.header
|
||||
if options.header != nil {
|
||||
for k, v := range options.header {
|
||||
req.Header.Add(k, strings.Join(v, " "))
|
||||
}
|
||||
}
|
||||
|
||||
if options.masterMeta && conf.SystemConfig.Mode == "master" {
|
||||
req.Header.Add("X-Site-Url", model.GetSiteURL().String())
|
||||
req.Header.Add("X-Site-Id", model.GetSettingByName("siteID"))
|
||||
req.Header.Add("X-Cloudreve-Version", conf.BackendVersion)
|
||||
}
|
||||
|
||||
if options.slaveNodeID != "" && conf.SystemConfig.Mode == "slave" {
|
||||
req.Header.Add("X-Node-Id", options.slaveNodeID)
|
||||
}
|
||||
|
||||
if options.contentLength != -1 {
|
||||
req.ContentLength = options.contentLength
|
||||
}
|
||||
|
||||
// 签名请求
|
||||
if options.sign != nil {
|
||||
auth.SignRequest(options.sign, req, options.signTTL)
|
||||
switch method {
|
||||
case "PUT", "POST", "PATCH":
|
||||
auth.SignRequest(options.sign, req, options.signTTL)
|
||||
default:
|
||||
if resURL, err := auth.SignURI(options.sign, req.URL.String(), options.signTTL); err == nil {
|
||||
req.URL = resURL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 发送请求
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
)
|
||||
|
||||
// TODO: move to slave pkg
|
||||
// RemoteCallback 发送远程存储策略上传回调请求
|
||||
func RemoteCallback(url string, body serializer.UploadCallback) error {
|
||||
callbackBody, err := json.Marshal(struct {
|
||||
|
||||
Reference in New Issue
Block a user