1
0
mirror of https://github.com/rclone/rclone.git synced 2025-12-10 05:13:45 +00:00

Compare commits

...

6 Commits

Author SHA1 Message Date
dependabot[bot]
0dde16d776 build: bump DavidAnson/markdownlint-cli2-action from 20 to 22
Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 20 to 22.
- [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases)
- [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v20...v22)

---
updated-dependencies:
- dependency-name: DavidAnson/markdownlint-cli2-action
  dependency-version: '22'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-12-09 22:06:02 +00:00
jhasse-shade
175d4bc553 Add Shade backend 2025-12-09 17:08:57 +00:00
Nick Craig-Wood
4851f1796c log: fix backtrace not going to the --log-file #9014
Before the log re-organisation in:

8d353039a6 log: add log rotation to --log-file

rclone would write any backtraces to the --log-file which was very
convenient for users.

This got accidentally disabled due to a typo which meant backtraces
started going to stderr even if --log-file was supplied.

This fixes the problem.
2025-12-09 16:35:07 +00:00
Nick Craig-Wood
4ff8899b2c build: fix lint warning after linter upgrade 2025-12-09 16:15:17 +00:00
Nick Craig-Wood
8f29a0b0a1 Add Jonas Tingeborn to contributors 2025-12-09 16:15:17 +00:00
Nick Craig-Wood
8b0e76e53b Add Tingsong Xu to contributors 2025-12-09 16:15:17 +00:00
17 changed files with 1635 additions and 3 deletions

View File

@@ -283,7 +283,7 @@ jobs:
run: govulncheck ./...
- name: Check Markdown format
uses: DavidAnson/markdownlint-cli2-action@v20
uses: DavidAnson/markdownlint-cli2-action@v22
with:
globs: |
CONTRIBUTING.md

View File

@@ -109,6 +109,7 @@ directories to and from different cloud storage providers.
- Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
- Servercore Object Storage [:page_facing_up:](https://rclone.org/s3/#servercore)
- SFTP [:page_facing_up:](https://rclone.org/sftp/)
- Shade [:page_facing_up:](https://rclone.org/shade/)
- SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
- Spectra Logic [:page_facing_up:](https://rclone.org/s3/#spectralogic)
- StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)

View File

@@ -55,6 +55,7 @@ import (
_ "github.com/rclone/rclone/backend/s3"
_ "github.com/rclone/rclone/backend/seafile"
_ "github.com/rclone/rclone/backend/sftp"
_ "github.com/rclone/rclone/backend/shade"
_ "github.com/rclone/rclone/backend/sharefile"
_ "github.com/rclone/rclone/backend/sia"
_ "github.com/rclone/rclone/backend/smb"

View File

@@ -0,0 +1,27 @@
// Package api has type definitions for shade
package api
// ListDirResponse -------------------------------------------------
// Format from shade api
type ListDirResponse struct {
Type string `json:"type"` // "file" or "tree"
Path string `json:"path"` // Full path including root
Ino int `json:"ino"` // inode number
Mtime int64 `json:"mtime"` // Modified time in milliseconds
Ctime int64 `json:"ctime"` // Created time in milliseconds
Size int64 `json:"size"` // Size in bytes
Hash string `json:"hash"` // MD5 hash
Draft bool `json:"draft"` // Whether this is a draft file
}
// PartURL Type for multipart upload/download
type PartURL struct {
URL string `json:"url"`
Headers map[string]string `json:"headers,omitempty"`
}
// CompletedPart Type for completed parts when making a multipart upload.
type CompletedPart struct {
ETag string
PartNumber int32
}

1017
backend/shade/shade.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,21 @@
package shade_test
import (
"testing"
"github.com/rclone/rclone/backend/shade"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
name := "TestShade"
fstests.Run(t, &fstests.Opt{
RemoteName: name + ":",
NilObject: (*shade.Object)(nil),
SkipInvalidUTF8: true,
ExtraConfig: []fstests.ExtraConfigItem{
{Name: name, Key: "eventually_consistent_delay", Value: "7"},
},
})
}

336
backend/shade/upload.go Normal file
View File

@@ -0,0 +1,336 @@
//multipart upload for shade
package shade
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"path"
"sort"
"sync"
"github.com/rclone/rclone/backend/shade/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/rest"
)
var warnStreamUpload sync.Once
type shadeChunkWriter struct {
initToken string
chunkSize int64
size int64
f *Fs
o *Object
completedParts []api.CompletedPart
completedPartsMu sync.Mutex
}
// uploadMultipart handles multipart upload for larger files
func (o *Object) uploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.Reader, options ...fs.OpenOption) error {
chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
Open: o.fs,
OpenOptions: options,
})
if err != nil {
return err
}
var shadeWriter = chunkWriter.(*shadeChunkWriter)
o.size = shadeWriter.size
return nil
}
// OpenChunkWriter returns the chunk size and a ChunkWriter
//
// Pass in the remote and the src object
// You can also use options to hint at the desired chunk size
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
// Temporary Object under construction
o := &Object{
fs: f,
remote: remote,
}
uploadParts := f.opt.MaxUploadParts
if uploadParts < 1 {
uploadParts = 1
} else if uploadParts > maxUploadParts {
uploadParts = maxUploadParts
}
size := src.Size()
fs.FixRangeOption(options, size)
// calculate size of parts
chunkSize := f.opt.ChunkSize
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 64 MB). With a maximum number of parts (10,000) this will be a file of
// 640 GB.
if size == -1 {
warnStreamUpload.Do(func() {
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
chunkSize, fs.SizeSuffix(int64(chunkSize)*int64(uploadParts)))
})
} else {
chunkSize = chunksize.Calculator(src, size, uploadParts, chunkSize)
}
token, err := o.fs.refreshJWTToken(ctx)
if err != nil {
return info, nil, fmt.Errorf("failed to get token: %w", err)
}
err = f.ensureParentDirectories(ctx, remote)
if err != nil {
return info, nil, fmt.Errorf("failed to ensure parent directories: %w", err)
}
fullPath := remote
if f.root != "" {
fullPath = path.Join(f.root, remote)
}
// Initiate multipart upload
type initRequest struct {
Path string `json:"path"`
PartSize int64 `json:"partSize"`
}
reqBody := initRequest{
Path: fullPath,
PartSize: int64(chunkSize),
}
var initResp struct {
Token string `json:"token"`
}
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/%s/upload/multipart", o.fs.drive),
RootURL: o.fs.endpoint,
ExtraHeaders: map[string]string{
"Authorization": "Bearer " + token,
},
Options: options,
}
err = o.fs.pacer.Call(func() (bool, error) {
res, err := o.fs.srv.CallJSON(ctx, &opts, reqBody, &initResp)
if err != nil {
return res != nil && res.StatusCode == http.StatusTooManyRequests, err
}
return false, nil
})
if err != nil {
return info, nil, fmt.Errorf("failed to initiate multipart upload: %w", err)
}
chunkWriter := &shadeChunkWriter{
initToken: initResp.Token,
chunkSize: int64(chunkSize),
size: size,
f: f,
o: o,
}
info = fs.ChunkWriterInfo{
ChunkSize: int64(chunkSize),
Concurrency: f.opt.Concurrency,
LeavePartsOnError: false,
}
return info, chunkWriter, err
}
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
token, err := s.f.refreshJWTToken(ctx)
if err != nil {
return 0, err
}
// Read chunk
var chunk bytes.Buffer
n, err := io.Copy(&chunk, reader)
if n == 0 {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("failed to read chunk: %w", err)
}
// Get presigned URL for this part
var partURL api.PartURL
partOpts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/%s/upload/multipart/part/%d?token=%s", s.f.drive, chunkNumber+1, url.QueryEscape(s.initToken)),
RootURL: s.f.endpoint,
ExtraHeaders: map[string]string{
"Authorization": "Bearer " + token,
},
}
err = s.f.pacer.Call(func() (bool, error) {
res, err := s.f.srv.CallJSON(ctx, &partOpts, nil, &partURL)
if err != nil {
return res != nil && res.StatusCode == http.StatusTooManyRequests, err
}
return false, nil
})
if err != nil {
return 0, fmt.Errorf("failed to get part URL: %w", err)
}
opts := rest.Opts{
Method: "PUT",
RootURL: partURL.URL,
Body: &chunk,
ContentType: "",
ContentLength: &n,
}
// Add headers
var uploadRes *http.Response
if len(partURL.Headers) > 0 {
opts.ExtraHeaders = make(map[string]string)
for k, v := range partURL.Headers {
opts.ExtraHeaders[k] = v
}
}
err = s.f.pacer.Call(func() (bool, error) {
uploadRes, err = s.f.srv.Call(ctx, &opts)
if err != nil {
return uploadRes != nil && uploadRes.StatusCode == http.StatusTooManyRequests, err
}
return false, nil
})
if err != nil {
return 0, fmt.Errorf("failed to upload part %d: %w", chunk, err)
}
if uploadRes.StatusCode != http.StatusOK && uploadRes.StatusCode != http.StatusCreated {
body, _ := io.ReadAll(uploadRes.Body)
fs.CheckClose(uploadRes.Body, &err)
return 0, fmt.Errorf("part upload failed with status %d: %s", uploadRes.StatusCode, string(body))
}
// Get ETag from response
etag := uploadRes.Header.Get("ETag")
fs.CheckClose(uploadRes.Body, &err)
s.completedPartsMu.Lock()
defer s.completedPartsMu.Unlock()
s.completedParts = append(s.completedParts, api.CompletedPart{
PartNumber: int32(chunkNumber + 1),
ETag: etag,
})
return n, nil
}
// Close complete chunked writer finalising the file.
func (s *shadeChunkWriter) Close(ctx context.Context) error {
// Complete multipart upload
sort.Slice(s.completedParts, func(i, j int) bool {
return s.completedParts[i].PartNumber < s.completedParts[j].PartNumber
})
type completeRequest struct {
Parts []api.CompletedPart `json:"parts"`
}
var completeBody completeRequest
if s.completedParts == nil {
completeBody = completeRequest{Parts: []api.CompletedPart{}}
} else {
completeBody = completeRequest{Parts: s.completedParts}
}
token, err := s.f.refreshJWTToken(ctx)
if err != nil {
return err
}
completeOpts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/%s/upload/multipart/complete?token=%s", s.f.drive, url.QueryEscape(s.initToken)),
RootURL: s.f.endpoint,
ExtraHeaders: map[string]string{
"Authorization": "Bearer " + token,
},
}
var response http.Response
err = s.f.pacer.Call(func() (bool, error) {
res, err := s.f.srv.CallJSON(ctx, &completeOpts, completeBody, &response)
if err != nil && res == nil {
return false, err
}
if res.StatusCode == http.StatusTooManyRequests {
return true, err // Retry on 429
}
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
body, _ := io.ReadAll(res.Body)
return false, fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))
}
return false, nil
})
if err != nil {
return fmt.Errorf("failed to complete multipart upload: %w", err)
}
return nil
}
// Abort chunk write
//
// You can and should call Abort without calling Close.
func (s *shadeChunkWriter) Abort(ctx context.Context) error {
token, err := s.f.refreshJWTToken(ctx)
if err != nil {
return err
}
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/%s/upload/abort/multipart?token=%s", s.f.drive, url.QueryEscape(s.initToken)),
RootURL: s.f.endpoint,
ExtraHeaders: map[string]string{
"Authorization": "Bearer " + token,
},
}
err = s.f.pacer.Call(func() (bool, error) {
res, err := s.f.srv.Call(ctx, &opts)
if err != nil {
fs.Debugf(s.f, "Failed to abort multipart upload: %v", err)
return false, nil // Don't retry abort
}
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
fs.Debugf(s.f, "Abort returned status %d", res.StatusCode)
}
return false, nil
})
if err != nil {
return fmt.Errorf("failed to abort multipart upload: %w", err)
}
return nil
}

View File

@@ -84,6 +84,7 @@ docs = [
"protondrive.md",
"seafile.md",
"sftp.md",
"shade.md",
"smb.md",
"storj.md",
"sugarsync.md",

View File

@@ -202,6 +202,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
{{< provider name="Servercore Object Storage" home="https://servercore.com/services/object-storage/" config="/s3/#servercore" >}}
{{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
{{< provider name="Shade" home="https://shade.inc" config="/shade/" >}}
{{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}}
{{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}}
{{< provider name="Spectra Logic" home="https://spectralogic.com/blackpearl-nearline-object-gateway/" config="/s3/#spectralogic" >}}

View File

@@ -1055,3 +1055,5 @@ put them back in again. -->
- Vladislav Tropnikov <vtr.name@gmail.com>
- Leo <i@hardrain980.com>
- Johannes Rothe <mail@johannes-rothe.de>
- Tingsong Xu <tingsong.xu@rightcapital.com>
- Jonas Tingeborn <134889+jojje@users.noreply.github.com>

View File

@@ -82,6 +82,7 @@ See the following for detailed instructions for
- [rsync.net](/sftp/#rsync-net)
- [Seafile](/seafile/)
- [SFTP](/sftp/)
- [Shade](/shade/)
- [Sia](/sia/)
- [SMB](/smb/)
- [Storj](/storj/)

View File

@@ -59,6 +59,7 @@ Here is an overview of the major features of each cloud storage system.
| Quatrix by Maytech | - | R/W | No | No | - | - |
| Seafile | - | - | No | No | - | - |
| SFTP | MD5, SHA1 ² | DR/W | Depends | No | - | - |
| Shade | - | - | Yes | No | - | - |
| Sia | - | - | No | No | - | - |
| SMB | - | R/W | Yes | No | - | - |
| SugarSync | - | - | No | No | - | - |

218
docs/content/shade.md Normal file
View File

@@ -0,0 +1,218 @@
# {{< icon "fa fa-moon" >}} Shade
This is a backend for the [Shade](https://shade.inc/) platform
## About Shade
[Shade](https://shade.inc/) is an AI-powered cloud NAS that makes your cloud files behave like a local drive, optimized for media and creative workflows. It provides fast, secure access with natural-language search, easy sharing, and scalable cloud storage.
## Accounts & Pricing
To use this backend, you need to [create a free account](https://app.shade.inc/) on Shade. You can start with a free account and get 20GB of storage for free.
## Usage
Paths are specified as `remote:path`
Paths may be as deep as required, e.g. `remote:directory/subdirectory`.
## Configuration
Here is an example of making a Shade configuration.
First, create a [create a free account](https://app.shade.inc/) account and choose a plan.
You will need to log in and get the `API Key` and `Drive ID` for your account from the settings section of your account and created drive respectively.
Now run
`rclone config`
Follow this interactive process:
```sh
$ rclone config
e) Edit existing remote
n) New remote
d) Delete remote
r) Rename remote
c) Copy remote
s) Set configuration password
q) Quit config
e/n/d/r/c/s/q> n
Enter name for new remote.
name> Shade
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
[OTHER OPTIONS]
xx / Shade FS
\ (shade)
[OTHER OPTIONS]
Storage> xx
Option drive_id.
The ID of your drive, see this in the drive settings. Individual rclone configs must be made per drive.
Enter a value.
drive_id> [YOUR_ID]
Option api_key.
An API key for your account.
Enter a value.
api_key> [YOUR_API_KEY]
Edit advanced config?
y) Yes
n) No (default)
y/n> n
Configuration complete.
Options:
- type: shade
- drive_id: [YOUR_ID]
- api_key: [YOUR_API_KEY]
Keep this "Shade" remote?
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
### Modification times and hashes
Shade does not support hashes and writing mod times.
### Transfers
Shade uses multipart uploads by default. This means that files will be chunked and sent up to Shade concurrently. In order to configure how many simultaneous uploads you want to use, upload the 'concurrency' option in the advanced config section. Note that this uses more memory and initiates more http requests.
### Deleting files
Please note that when deleting files in Shade via rclone it will delete the file instantly, instead of sending it to the trash. This means that it will not be recoverable.
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/box/box.go then run make backenddocs" >}}
### Standard options
Here are the Standard options specific to shade (Shade FS).
#### --shade-drive-id
The ID of your drive, see this in the drive settings. Individual rclone configs must be made per drive.
Properties:
- Config: drive_id
- Env Var: RCLONE_SHADE_DRIVE_ID
- Type: string
- Required: true
#### --shade-api-key
An API key for your account. You can find this under Settings > API Keys
Properties:
- Config: api_key
- Env Var: RCLONE_SHADE_API_KEY
- Type: string
- Required: true
### Advanced options
Here are the Advanced options specific to shade (Shade FS).
#### --shade-endpoint
Endpoint for the service.
Leave blank normally.
Properties:
- Config: endpoint
- Env Var: RCLONE_SHADE_ENDPOINT
- Type: string
- Required: false
#### --shade-chunk-size
Chunk size to use for uploading.
Any files larger than this will be uploaded in chunks of this size.
Note that this is stored in memory per transfer, so increasing it will
increase memory usage.
Minimum is 5MB, maximum is 5GB.
Properties:
- Config: chunk_size
- Env Var: RCLONE_SHADE_CHUNK_SIZE
- Type: SizeSuffix
- Default: 64Mi
#### --shade-encoding
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Properties:
- Config: encoding
- Env Var: RCLONE_SHADE_ENCODING
- Type: Encoding
- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
#### --shade-description
Description of the remote.
Properties:
- Config: description
- Env Var: RCLONE_SHADE_DESCRIPTION
- Type: string
- Required: false
{{< rem autogenerated options stop >}}
## Limitations
Note that Shade is case insensitive so you can't have a file called
"Hello.doc" and one called "hello.doc".
Shade only supports filenames up to 255 characters in length.
`rclone about` is not supported by the Shade backend. Backends without
this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
## Backend commands
Here are the commands specific to the shade backend.
Run them with
rclone backend COMMAND remote:
The help below will explain what arguments each command takes.
See the [backend](/commands/rclone_backend/) command for more
info on how to pass options and arguments.
These can be run on a running backend using the rc command
[backend/command](/rc/#backend-command).

View File

@@ -107,6 +107,7 @@
<a class="dropdown-item" href="/seafile/"><i class="fa fa-server fa-fw"></i> Seafile</a>
<a class="dropdown-item" href="/sftp/"><i class="fa fa-server fa-fw"></i> SFTP</a>
<a class="dropdown-item" href="/sia/"><i class="fa fa-globe fa-fw"></i> Sia</a>
<a class="dropdown-item" href="/shade/"><i class="fa fa-moon fa-fw"></i> Shade</a>
<a class="dropdown-item" href="/smb/"><i class="fa fa-server fa-fw"></i> SMB / CIFS</a>
<a class="dropdown-item" href="/storj/"><i class="fas fa-dove fa-fw"></i> Storj</a>
<a class="dropdown-item" href="/sugarsync/"><i class="fas fa-dove fa-fw"></i> SugarSync</a>

View File

@@ -372,7 +372,7 @@ func (p *pipedInput) Read(b []byte) (int, error) {
return p.Reader.Read(b)
}
func (_ *pipedInput) Seek(_ int64, _ int) (int64, error) {
func (*pipedInput) Seek(int64, int) (int64, error) {
return 0, fmt.Errorf("Seek not supported")
}

View File

@@ -209,7 +209,7 @@ func InitLogging() {
// Log file output
if Opt.File != "" {
var w io.Writer
if Opt.MaxSize == 0 {
if Opt.MaxSize < 0 {
// No log rotation - just open the file as normal
// We'll capture tracebacks like this too.
f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)

View File

@@ -662,6 +662,10 @@ backends:
ignoretests:
- cmd/bisync
- cmd/gitannex
- backend: "shade"
remote: "TestShade:"
fastlist: false
- backend: "archive"
remote: "TestArchive:"
fastlist: false