mirror of
https://github.com/rclone/rclone.git
synced 2025-12-06 00:03:32 +00:00
Add pixeldrain backend
This commit adds support for pixeldrain's experimental filesystem API.
This commit is contained in:
Fornax
GitHub
397
backend/pixeldrain/api_client.go
Normal file
397
backend/pixeldrain/api_client.go
Normal file
@@ -0,0 +1,397 @@
|
||||
package pixeldrain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// FilesystemPath is the object which is returned from the pixeldrain API when
|
||||
// running the stat command on a path. It includes the node information for all
|
||||
// the members of the path and for all the children of the requested directory.
|
||||
type FilesystemPath struct {
|
||||
Path []FilesystemNode `json:"path"`
|
||||
BaseIndex int `json:"base_index"`
|
||||
Children []FilesystemNode `json:"children"`
|
||||
}
|
||||
|
||||
// Base returns the base node of the path, this is the node that the path points
|
||||
// to
|
||||
func (fsp *FilesystemPath) Base() FilesystemNode {
|
||||
return fsp.Path[fsp.BaseIndex]
|
||||
}
|
||||
|
||||
// FilesystemNode is a single node in the pixeldrain filesystem. Usually part of
|
||||
// a Path or Children slice. The Node is also returned as response from update
|
||||
// commands, if requested
|
||||
type FilesystemNode struct {
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
Created time.Time `json:"created"`
|
||||
Modified time.Time `json:"modified"`
|
||||
ModeOctal string `json:"mode_octal"`
|
||||
|
||||
// File params
|
||||
FileSize int64 `json:"file_size"`
|
||||
FileType string `json:"file_type"`
|
||||
SHA256Sum string `json:"sha256_sum"`
|
||||
|
||||
// ID is only filled in when the file/directory is publicly shared
|
||||
ID string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
// ChangeLog is a log of changes that happened in a filesystem. Changes returned
|
||||
// from the API are on chronological order from old to new. A change log can be
|
||||
// requested for any directory or file, but change logging needs to be enabled
|
||||
// with the update API before any log entries will be made. Changes are logged
|
||||
// for 24 hours after logging was enabled. Each time a change log is requested
|
||||
// the timer is reset to 24 hours.
|
||||
type ChangeLog []ChangeLogEntry
|
||||
|
||||
// ChangeLogEntry is a single entry in a directory's change log. It contains the
|
||||
// time at which the change occurred. The path relative to the requested
|
||||
// directory and the action that was performend (update, move or delete). In
|
||||
// case of a move operation the new path of the file is stored in the path_new
|
||||
// field
|
||||
type ChangeLogEntry struct {
|
||||
Time time.Time `json:"time"`
|
||||
Path string `json:"path"`
|
||||
PathNew string `json:"path_new"`
|
||||
Action string `json:"action"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// UserInfo contains information about the logged in user
|
||||
type UserInfo struct {
|
||||
Username string `json:"username"`
|
||||
Subscription SubscriptionType `json:"subscription"`
|
||||
StorageSpaceUsed int64 `json:"storage_space_used"`
|
||||
}
|
||||
|
||||
// SubscriptionType contains information about a subscription type. It's not the
|
||||
// active subscription itself, only the properties of the subscription. Like the
|
||||
// perks and cost
|
||||
type SubscriptionType struct {
|
||||
Name string `json:"name"`
|
||||
StorageSpace int64 `json:"storage_space"`
|
||||
}
|
||||
|
||||
// APIError is the error type returned by the pixeldrain API
|
||||
type APIError struct {
|
||||
StatusCode string `json:"value"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e APIError) Error() string { return e.StatusCode }
|
||||
|
||||
// Generalized errors which are caught in our own handlers and translated to
|
||||
// more specific errors from the fs package.
|
||||
var (
|
||||
errNotFound = errors.New("pd api: path not found")
|
||||
errExists = errors.New("pd api: node already exists")
|
||||
errAuthenticationFailed = errors.New("pd api: authentication failed")
|
||||
)
|
||||
|
||||
func apiErrorHandler(resp *http.Response) (err error) {
|
||||
var e APIError
|
||||
if err = json.NewDecoder(resp.Body).Decode(&e); err != nil {
|
||||
return fmt.Errorf("failed to parse error json: %w", err)
|
||||
}
|
||||
|
||||
// We close the body here so that the API handlers can be sure that the
|
||||
// response body is not still open when an error was returned
|
||||
if err = resp.Body.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close resp body: %w", err)
|
||||
}
|
||||
|
||||
if e.StatusCode == "path_not_found" {
|
||||
return errNotFound
|
||||
} else if e.StatusCode == "directory_not_empty" {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
} else if e.StatusCode == "node_already_exists" {
|
||||
return errExists
|
||||
} else if e.StatusCode == "authentication_failed" {
|
||||
return errAuthenticationFailed
|
||||
} else if e.StatusCode == "permission_denied" {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests.
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err deserve to be
|
||||
// retried. It returns the err as a convenience so it can be used as the return
|
||||
// value in the pacer function
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
// paramsFromMetadata turns the fs.Metadata into instructions the pixeldrain API
|
||||
// can understand.
|
||||
func paramsFromMetadata(meta fs.Metadata) (params url.Values) {
|
||||
params = make(url.Values)
|
||||
|
||||
if modified, ok := meta["mtime"]; ok {
|
||||
params.Set("modified", modified)
|
||||
}
|
||||
if created, ok := meta["btime"]; ok {
|
||||
params.Set("created", created)
|
||||
}
|
||||
if mode, ok := meta["mode"]; ok {
|
||||
params.Set("mode", mode)
|
||||
}
|
||||
if shared, ok := meta["shared"]; ok {
|
||||
params.Set("shared", shared)
|
||||
}
|
||||
if loggingEnabled, ok := meta["logging_enabled"]; ok {
|
||||
params.Set("logging_enabled", loggingEnabled)
|
||||
}
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
// nodeToObject converts a single FilesystemNode API response to an object. The
|
||||
// node is usually a single element from a directory listing
|
||||
func (f *Fs) nodeToObject(node FilesystemNode) (o *Object) {
|
||||
// Trim the path prefix. The path prefix is hidden from rclone during all
|
||||
// operations. Saving it here would confuse rclone a lot. So instead we
|
||||
// strip it here and add it back for every API request we need to perform
|
||||
node.Path = strings.TrimPrefix(node.Path, f.pathPrefix)
|
||||
return &Object{fs: f, base: node}
|
||||
}
|
||||
|
||||
func (f *Fs) nodeToDirectory(node FilesystemNode) fs.DirEntry {
|
||||
return fs.NewDir(strings.TrimPrefix(node.Path, f.pathPrefix), node.Modified).SetID(node.ID)
|
||||
}
|
||||
|
||||
func (f *Fs) escapePath(p string) (out string) {
|
||||
// Add the path prefix, encode all the parts and combine them together
|
||||
var parts = strings.Split(f.pathPrefix+p, "/")
|
||||
for i := range parts {
|
||||
parts[i] = url.PathEscape(parts[i])
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func (f *Fs) put(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
body io.Reader,
|
||||
meta fs.Metadata,
|
||||
options []fs.OpenOption,
|
||||
) (node FilesystemNode, err error) {
|
||||
var params = paramsFromMetadata(meta)
|
||||
|
||||
// Tell the server to automatically create parent directories if they don't
|
||||
// exist yet
|
||||
params.Set("make_parents", "true")
|
||||
|
||||
return node, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: f.escapePath(path),
|
||||
Body: body,
|
||||
Parameters: params,
|
||||
Options: options,
|
||||
},
|
||||
nil,
|
||||
&node,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) read(ctx context.Context, path string, options []fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &rest.Opts{
|
||||
Method: "GET",
|
||||
Path: f.escapePath(path),
|
||||
Options: options,
|
||||
})
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Body, err
|
||||
}
|
||||
|
||||
func (f *Fs) stat(ctx context.Context, path string) (fsp FilesystemPath, err error) {
|
||||
return fsp, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "GET",
|
||||
Path: f.escapePath(path),
|
||||
// To receive node info from the pixeldrain API you need to add the
|
||||
// ?stat query. Without it pixeldrain will return the file contents
|
||||
// in the URL points to a file
|
||||
Parameters: url.Values{"stat": []string{""}},
|
||||
},
|
||||
nil,
|
||||
&fsp,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) changeLog(ctx context.Context, start, end time.Time) (changeLog ChangeLog, err error) {
|
||||
return changeLog, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "GET",
|
||||
Path: f.escapePath(""),
|
||||
Parameters: url.Values{
|
||||
"change_log": []string{""},
|
||||
"start": []string{start.Format(time.RFC3339Nano)},
|
||||
"end": []string{end.Format(time.RFC3339Nano)},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
&changeLog,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) update(ctx context.Context, path string, fields fs.Metadata) (node FilesystemNode, err error) {
|
||||
var params = paramsFromMetadata(fields)
|
||||
params.Set("action", "update")
|
||||
|
||||
return node, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.escapePath(path),
|
||||
MultipartParams: params,
|
||||
},
|
||||
nil,
|
||||
&node,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) mkdir(ctx context.Context, dir string) (err error) {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.escapePath(dir),
|
||||
MultipartParams: url.Values{"action": []string{"mkdirall"}},
|
||||
NoResponse: true,
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
var errIncompatibleSourceFS = errors.New("source filesystem is not the same as target")
|
||||
|
||||
// Renames a file on the server side. Can be used for both directories and files
|
||||
func (f *Fs) rename(ctx context.Context, src fs.Fs, from, to string, meta fs.Metadata) (node FilesystemNode, err error) {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
// This is not a pixeldrain FS, can't move
|
||||
return node, errIncompatibleSourceFS
|
||||
} else if srcFs.opt.RootFolderID != f.opt.RootFolderID {
|
||||
// Path is not in the same root dir, can't move
|
||||
return node, errIncompatibleSourceFS
|
||||
}
|
||||
|
||||
var params = paramsFromMetadata(meta)
|
||||
params.Set("action", "rename")
|
||||
|
||||
// The target is always in our own filesystem so here we use our
|
||||
// own pathPrefix
|
||||
params.Set("target", f.pathPrefix+to)
|
||||
|
||||
// Create parent directories if the parent directory of the file
|
||||
// does not exist yet
|
||||
params.Set("make_parents", "true")
|
||||
|
||||
return node, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "POST",
|
||||
// Important: We use the source FS path prefix here
|
||||
Path: srcFs.escapePath(from),
|
||||
MultipartParams: params,
|
||||
},
|
||||
nil,
|
||||
&node,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) delete(ctx context.Context, path string, recursive bool) (err error) {
|
||||
var params url.Values
|
||||
if recursive {
|
||||
// Tell the server to recursively delete all child files
|
||||
params = url.Values{"recursive": []string{"true"}}
|
||||
}
|
||||
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: f.escapePath(path),
|
||||
Parameters: params,
|
||||
NoResponse: true,
|
||||
},
|
||||
nil, nil,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) userInfo(ctx context.Context) (user UserInfo, err error) {
|
||||
return user, f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(
|
||||
ctx,
|
||||
&rest.Opts{
|
||||
Method: "GET",
|
||||
// The default RootURL points at the filesystem endpoint. We can't
|
||||
// use that to request user information. So here we override it to
|
||||
// the user endpoint
|
||||
RootURL: f.opt.APIURL + "/user",
|
||||
},
|
||||
nil,
|
||||
&user,
|
||||
)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user