1
0
mirror of https://github.com/rclone/rclone.git synced 2025-12-06 00:03:32 +00:00

Compare commits

...

22 Commits

Author SHA1 Message Date
Nick Craig-Wood
0430163180 Version v1.53.1 2020-09-13 09:40:14 +01:00
Evan Harris
09a0dc1600 opendrive: Do not retry 400 errors
This type of error is unlikely to be an error that can be resolved by a retry,
and is triggered in #2296 by files with a timestamp before the unix epoch.
2020-09-12 12:49:57 +01:00
Evan Harris
dd11778ac6 docs: Updated mount command to reflect that it requires Go 1.13 or newer 2020-09-12 12:49:57 +01:00
wjielai
f36cbe5194 docs: add Tencent COS to s3 provider list - fixes #4468
* add Tencent COS to s3 provider list.

Co-authored-by: wjielai <wjielai@tencent.com>
2020-09-12 12:49:57 +01:00
edwardxml
82a383588b docs: Add full stops for consistency in rclone --help
closes #4560 closes #4561 closes #4562 closes #4563 closes #4564
2020-09-12 12:49:56 +01:00
albertony
8ae4d2cffe docs/jottacloud: mention that uploads from local disk will not need to cache files to disk for md5 calculation 2020-09-12 12:48:57 +01:00
Nick Craig-Wood
0f895c0697 vfs,local: Log an ERROR if we fail to set the file to be sparse
See: https://forum.rclone.org/t/rclone-1-53-release/18880/73
2020-09-11 15:37:26 +01:00
Nick Craig-Wood
937dd7fa1f docs: note --log-file does append 2020-09-08 16:13:49 +01:00
Nick Craig-Wood
33869387d1 build: fix architecture name in ARMv7 build - fixes #4571
After introducing the arm-v7 build we are accidentally making debs
and rpms with the architecture arm-v7.

This fixes the problem by stripping the version off.
2020-09-08 16:09:52 +01:00
Nick Craig-Wood
3ec8e304b3 accounting: remove new line from end of --stats-one-line display 2020-09-08 16:09:50 +01:00
Tim Gallant
e62362094e drive: adds special oauth help test - fixes #4555 2020-09-07 12:49:30 +01:00
Nick Craig-Wood
6a0398211d build: don't explicitly set ARM version to fix ARMv5 build #4553
This partially reverts commit f71f6c57d7.
2020-09-07 12:39:39 +01:00
Nick Craig-Wood
e5a53d4c65 check: fix docs
See: https://forum.rclone.org/t/possible-issue-with-documention/18926
2020-09-07 12:12:10 +01:00
Nick Craig-Wood
59d5767a07 check: add back missing --download flag - fixes #4565
This was accidentally removed when refactoring check and cryptcheck in

8b6f2bbb4b check,cryptcheck: add reporting of filenames for same/missing/changed #3264
2020-09-05 09:31:55 +01:00
Nick Craig-Wood
087b5788e2 build: fix "Illegal instruction" error for ARMv6 builds - fixes #4553
Before this change we used `go build -i` to build the releases in parallel.

However this causes the ARMv6 and ARMv7 build to get mixed up somehow,
causing an illegal instruction when running rclone binaries on ARMv6.

See go bug: https://github.com/golang/go/issues/41223

This removes the -i which should have no effect on build times on the
CI and appears to fix the problem.
2020-09-04 16:33:22 +01:00
Nick Craig-Wood
d944bfd936 build: explicitly set ARM version to fix build #4553 2020-09-04 16:33:22 +01:00
Nick Craig-Wood
d780fcf317 docs: fix formatting of rc docs page
See: https://forum.rclone.org/t/rclone-1-53-release/18880/24
2020-09-03 11:53:53 +01:00
Nick Craig-Wood
0a9b8eac80 build: update build for stable branch 2020-09-03 11:29:33 +01:00
Nick Craig-Wood
1272a8f9a5 Start v1.53.1-DEV development 2020-09-03 11:28:43 +01:00
Nick Craig-Wood
0b40eaedaf build: include vendor tar ball in release and fix startdev 2020-09-03 10:59:17 +01:00
Nick Craig-Wood
8340ff4fb9 vfs: fix spurious error "vfs cache: failed to _ensure cache EOF"
Before this change the error message was produced for every file which
was confusing users.

After this change we check for EOF and return from ReadAt at that
point.

See: https://forum.rclone.org/t/rclone-1-53-release/18880/10
2020-09-03 10:43:41 +01:00
Nick Craig-Wood
f5abc168ed docs: fix download links 2020-09-02 17:58:07 +01:00
49 changed files with 33807 additions and 25315 deletions

17395
MANUAL.html generated

File diff suppressed because one or more lines are too long

315
MANUAL.md generated
View File

@@ -1,6 +1,6 @@
% rclone(1) User Manual
% Nick Craig-Wood
% Sep 02, 2020
% Sep 13, 2020
# Rclone syncs your files to cloud storage
@@ -146,6 +146,7 @@ WebDAV or S3, that work out of the box.)
- StackPath
- SugarSync
- Tardigrade
- Tencent Cloud Object Storage (COS)
- Wasabi
- WebDAV
- Yandex Disk
@@ -503,7 +504,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
# rclone copy
Copy files from source to dest, skipping already copied
Copy files from source to dest, skipping already copied.
## Synopsis
@@ -833,7 +834,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
@@ -859,6 +860,7 @@ rclone check source:path dest:path [flags]
```
--combined string Make a combined report of changes to this file
--differ string Report all non-matching files to this file
--download Check by downloading rather than with hash.
--error string Report all files with errors (hashing or reading) to this file
-h, --help help for check
--match string Report all matching files to this file
@@ -1191,7 +1193,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
# rclone cleanup
Clean up the remote if possible
Clean up the remote if possible.
## Synopsis
@@ -1915,7 +1917,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
# rclone copyto
Copy files from source to dest, skipping already copied
Copy files from source to dest, skipping already copied.
## Synopsis
@@ -2040,7 +2042,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
@@ -2434,7 +2436,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
# rclone lsf
List directories and objects in remote:path formatted for parsing
List directories and objects in remote:path formatted for parsing.
## Synopsis
@@ -2744,6 +2746,9 @@ Stopping the mount manually:
# OS X
umount /path/to/local/mount
**Note**: As of `rclone` 1.52.2, `rclone mount` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
## Installing on Windows
To run rclone mount on Windows, you will need to
@@ -3052,6 +3057,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -3289,7 +3299,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
# rclone obscure
Obscure password for use in the rclone config file
Obscure password for use in the rclone config file.
## Synopsis
@@ -3754,6 +3764,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -4056,6 +4071,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -4514,6 +4534,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -5035,6 +5060,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -5502,6 +5532,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -6502,6 +6537,8 @@ This can be useful for tracking down problems with syncs in
combination with the `-v` flag. See the [Logging section](#logging)
for more info.
If FILE exists then rclone will append to it.
Note that if you are using the `logrotate` program to manage rclone's
logs, then you should use the `copytruncate` option as rclone doesn't
have a signal to rotate logs.
@@ -8893,6 +8930,8 @@ OR
"result": "<Raw command line output>"
}
```
**Authentication is required for this call.**
### core/gc: Runs a garbage collection. {#core-gc}
@@ -9568,7 +9607,7 @@ This allows you to remove a plugin using it's name
This takes parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format `author`/`plugin_name`
Eg
@@ -9582,7 +9621,7 @@ This allows you to remove a plugin using it's name
This takes the following parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format `author`/`plugin_name`
Eg
@@ -10527,7 +10566,7 @@ These flags are available for every command.
--use-json-log Use json log format.
--use-mmap Use mmap allocator (see docs).
--use-server-modtime Use server modified time instead of object metadata
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.0")
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.1")
-v, --verbose count Print lots more stuff (repeat for more)
```
@@ -10626,7 +10665,7 @@ and may be set in the config file.
--drive-auth-owner-only Only consider files owned by the authenticated user.
--drive-auth-url string Auth server URL.
--drive-chunk-size SizeSuffix Upload chunk size. Must a power of 2 >= 256k. (default 8M)
--drive-client-id string OAuth Client Id
--drive-client-id string Google Application Client Id
--drive-client-secret string OAuth Client Secret
--drive-disable-http2 Disable drive using http2 (default true)
--drive-encoding MultiEncoder This sets the encoding for the backend. (default InvalidUtf8)
@@ -11475,6 +11514,7 @@ The S3 backend can be used with a number of different providers:
- Minio
- Scaleway
- StackPath
- Tencent Cloud Object Storage (COS)
- Wasabi
@@ -11912,7 +11952,7 @@ Vault API, so rclone cannot directly access Glacier Vaults.
### Standard Options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
#### --s3-provider
@@ -11943,6 +11983,8 @@ Choose your S3 provider.
- Scaleway Object Storage
- "StackPath"
- StackPath Object Storage
- "TencentCOS"
- Tencent Cloud Object Storage (COS)
- "Wasabi"
- Wasabi Object Storage
- "Other"
@@ -12296,6 +12338,54 @@ Endpoint for StackPath Object Storage.
#### --s3-endpoint
Endpoint for Tencent COS API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Type: string
- Default: ""
- Examples:
- "cos.ap-beijing.myqcloud.com"
- Beijing Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-shanghai.myqcloud.com"
- Shanghai Region.
- "cos.ap-guangzhou.myqcloud.com"
- Guangzhou Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-chengdu.myqcloud.com"
- Chengdu Region.
- "cos.ap-chongqing.myqcloud.com"
- Chongqing Region.
- "cos.ap-hongkong.myqcloud.com"
- Hong Kong (China) Region.
- "cos.ap-singapore.myqcloud.com"
- Singapore Region.
- "cos.ap-mumbai.myqcloud.com"
- Mumbai Region.
- "cos.ap-seoul.myqcloud.com"
- Seoul Region.
- "cos.ap-bangkok.myqcloud.com"
- Bangkok Region.
- "cos.ap-tokyo.myqcloud.com"
- Tokyo Region.
- "cos.na-siliconvalley.myqcloud.com"
- Silicon Valley Region.
- "cos.na-ashburn.myqcloud.com"
- Virginia Region.
- "cos.na-toronto.myqcloud.com"
- Toronto Region.
- "cos.eu-frankfurt.myqcloud.com"
- Frankfurt Region.
- "cos.eu-moscow.myqcloud.com"
- Moscow Region.
- "cos.accelerate.myqcloud.com"
- Use Tencent COS Accelerate Endpoint.
#### --s3-endpoint
Endpoint for S3 API.
Required when using an S3 clone.
@@ -12463,6 +12553,8 @@ doesn't copy the ACL from the source but rather writes a fresh one.
- Type: string
- Default: ""
- Examples:
- "default"
- Owner gets Full_CONTROL. No one else has access rights (default).
- "private"
- Owner gets FULL_CONTROL. No one else has access rights (default).
- "public-read"
@@ -12563,6 +12655,24 @@ The storage class to use when storing new objects in OSS.
#### --s3-storage-class
The storage class to use when storing new objects in Tencent COS.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Type: string
- Default: ""
- Examples:
- ""
- Default
- "STANDARD"
- Standard storage class
- "ARCHIVE"
- Archive storage mode.
- "STANDARD_IA"
- Infrequent access storage mode.
#### --s3-storage-class
The storage class to use when storing new objects in S3.
- Config: storage_class
@@ -12579,7 +12689,7 @@ The storage class to use when storing new objects in S3.
### Advanced Options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
#### --s3-bucket-acl
@@ -12800,7 +12910,7 @@ if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to
false - rclone will do this automatically based on the provider
setting.
@@ -13669,6 +13779,138 @@ d) Delete this remote
y/e/d> y
```
### Tencent COS {#tencent-cos}
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
To configure access to Tencent COS, follow the steps below:
1. Run `rclone config` and select `n` for a new remote.
```
rclone config
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
```
2. Give the name of the configuration. For example, name it 'cos'.
```
name> cos
```
3. Select `s3` storage.
```
Choose a number from below, or type in your own value
1 / 1Fichier
\ "fichier"
2 / Alias for an existing remote
\ "alias"
3 / Amazon Drive
\ "amazon cloud drive"
4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)
\ "s3"
[snip]
Storage> s3
```
4. Select `TencentCOS` provider.
```
Choose a number from below, or type in your own value
1 / Amazon Web Services (AWS) S3
\ "AWS"
[snip]
11 / Tencent Cloud Object Storage (COS)
\ "TencentCOS"
[snip]
provider> TencentCOS
```
5. Enter your SecretId and SecretKey of Tencent Cloud.
```
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Enter a boolean value (true or false). Press Enter for the default ("false").
Choose a number from below, or type in your own value
1 / Enter AWS credentials in the next step
\ "false"
2 / Get AWS credentials from the environment (env vars or IAM)
\ "true"
env_auth> 1
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
access_key_id> AKIDxxxxxxxxxx
AWS Secret Access Key (password)
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
secret_access_key> xxxxxxxxxxx
```
6. Select endpoint for Tencent COS. This is the standard endpoint for different region.
```
1 / Beijing Region.
\ "cos.ap-beijing.myqcloud.com"
2 / Nanjing Region.
\ "cos.ap-nanjing.myqcloud.com"
3 / Shanghai Region.
\ "cos.ap-shanghai.myqcloud.com"
4 / Guangzhou Region.
\ "cos.ap-guangzhou.myqcloud.com"
[snip]
endpoint> 4
```
7. Choose acl and storage class.
```
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Owner gets Full_CONTROL. No one else has access rights (default).
\ "default"
[snip]
acl> 1
The storage class to use when storing new objects in Tencent COS.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Default
\ ""
[snip]
storage_class> 1
Edit advanced config? (y/n)
y) Yes
n) No (default)
y/n> n
Remote config
--------------------
[cos]
type = s3
provider = TencentCOS
env_auth = false
access_key_id = xxx
secret_access_key = xxx
endpoint = cos.ap-guangzhou.myqcloud.com
acl = default
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
Current remotes:
Name Type
==== ====
cos s3
```
### Netease NOS ###
For Netease NOS configure as per the configurator `rclone config`
@@ -17997,8 +18239,10 @@ Here are the standard options specific to drive (Google Drive).
#### --drive-client-id
OAuth Client Id
Leave blank normally.
Google Application Client Id
Setting your own is recommended.
See https://rclone.org/drive/#making-your-own-client-id for how to create your own.
If you leave this blank, it will use an internal key which is low performance.
- Config: client_id
- Env Var: RCLONE_DRIVE_CLIENT_ID
@@ -19688,8 +19932,13 @@ flag.
Note that Jottacloud requires the MD5 hash before upload so if the
source does not have an MD5 checksum then the file will be cached
temporarily on disk (wherever the `TMPDIR` environment variable points
to) before it is uploaded. Small files will be cached in memory - see
to) before it is uploaded. Small files will be cached in memory - see
the [--jottacloud-md5-memory-limit](#jottacloud-md5-memory-limit) flag.
When uploading from local disk the source checksum is always available,
so this does not apply. Starting with rclone version 1.52 the same is
true for crypted remotes (in older versions the crypt backend would not
calculate hashes for uploads from local disk, so the Jottacloud
backend had to do it as described above).
#### Restricted filename characters
@@ -25432,6 +25681,36 @@ Options:
# Changelog
## v1.53.1 - 2020-09-13
[See commits](https://github.com/rclone/rclone/compare/v1.53.0...v1.53.1)
* Bug Fixes
* accounting: Remove new line from end of --stats-one-line display (Nick Craig-Wood)
* check
* Add back missing --download flag (Nick Craig-Wood)
* Fix docs (Nick Craig-Wood)
* docs
* Note --log-file does append (Nick Craig-Wood)
* Add full stops for consistency in rclone --help (edwardxml)
* Add Tencent COS to s3 provider list (wjielai)
* Updated mount command to reflect that it requires Go 1.13 or newer (Evan Harris)
* jottacloud: Mention that uploads from local disk will not need to cache files to disk for md5 calculation (albertony)
* Fix formatting of rc docs page (Nick Craig-Wood)
* build
* Include vendor tar ball in release and fix startdev (Nick Craig-Wood)
* Fix "Illegal instruction" error for ARMv6 builds (Nick Craig-Wood)
* Fix architecture name in ARMv7 build (Nick Craig-Wood)
* VFS
* Fix spurious error "vfs cache: failed to _ensure cache EOF" (Nick Craig-Wood)
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Local
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Drive
* Re-adds special oauth help text (Tim Gallant)
* Opendrive
* Do not retry 400 errors (Evan Harris)
## v1.53.0 - 2020-09-02
[See commits](https://github.com/rclone/rclone/compare/v1.52.0...v1.53.0)

17004
MANUAL.txt generated

File diff suppressed because it is too large Load Diff

View File

@@ -8,7 +8,8 @@ VERSION := $(shell cat VERSION)
# Last tag on this branch
LAST_TAG := $(shell git describe --tags --abbrev=0)
# Next version
NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
NEXT_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2+1,0}')
NEXT_PATCH_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2,$$3+1}')
# If we are working on a release, override branch to master
ifdef RELEASE_TAG
BRANCH := master
@@ -164,6 +165,11 @@ validate_website: website
tarball:
git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
vendorball:
go mod vendor
tar -zcf build/rclone-$(TAG)-vendor.tar.gz vendor
rm -rf vendor
sign_upload:
cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
@@ -239,7 +245,15 @@ startdev:
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_VERSION)" > VERSION
git commit -m "Start $(VERSION)-DEV development" fs/version.go
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
startstable:
@echo "Version is $(VERSION)"
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_PATCH_VERSION)" > VERSION
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
winzip:
zip -9 rclone-$(TAG).zip rclone.exe

View File

@@ -64,6 +64,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
* StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
* SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
* Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
* Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
* Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
* Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)

View File

@@ -9,7 +9,7 @@ This file describes how to make the various kinds of releases
## Making a release
* git checkout master
* git checkout master # see below for stable branch
* git pull
* git status - make sure everything is checked in
* Check GitHub actions build for master is Green
@@ -25,12 +25,13 @@ This file describes how to make the various kinds of releases
* # Wait for the GitHub builds to complete then...
* make fetch_binaries
* make tarball
* make vendorball
* make sign_upload
* make check_sign
* make upload
* make upload_website
* make upload_github
* make startdev
* make startdev # make startstable for stable branch
* # announce with forum post, twitter post, patreon post
Early in the next release cycle update the dependencies
@@ -41,62 +42,35 @@ Early in the next release cycle update the dependencies
* git add new files
* git commit -a -v
If `make update` fails with errors like this:
```
# github.com/cpuguy83/go-md2man/md2man
../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:11:16: undefined: blackfriday.EXTENSION_NO_INTRA_EMPHASIS
../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:12:16: undefined: blackfriday.EXTENSION_TABLES
```
Can be fixed with
* GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
* GO111MODULE=on go mod tidy
## Making a point release
If rclone needs a point release due to some horrendous bug:
First make the release branch. If this is a second point release then
this will be done already.
Set vars
* BASE_TAG=v1.XX # eg v1.52
* NEW_TAG=${BASE_TAG}.Y # eg v1.52.1
* echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
First make the release branch. If this is a second point release then
this will be done already.
* git branch ${BASE_TAG} ${BASE_TAG}-stable
* git co ${BASE_TAG}-stable
* make startstable
Now
* FIXME this is now broken with new semver layout - needs fixing
* FIXME the TAG=${NEW_TAG} shouldn't be necessary any more
* git co ${BASE_TAG}-stable
* git cherry-pick any fixes
* Test (see above)
* make NEXT_VERSION=${NEW_TAG} tag
* edit docs/content/changelog.md
* make TAG=${NEW_TAG} doc
* git commit -a -v -m "Version ${NEW_TAG}"
* git tag -d ${NEW_TAG}
* git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
* git push --tags -u origin ${BASE_TAG}-stable
* Wait for builds to complete
* make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
* make TAG=${NEW_TAG} tarball
* make TAG=${NEW_TAG} sign_upload
* make TAG=${NEW_TAG} check_sign
* make TAG=${NEW_TAG} upload
* make TAG=${NEW_TAG} upload_website
* make TAG=${NEW_TAG} upload_github
* NB this overwrites the current beta so we need to do this
* Do the steps as above
* make startstable
* NB this overwrites the current beta so we need to do this - FIXME is this true any more?
* git co master
* make VERSION=${NEW_TAG} startdev
* # cherry pick the changes to the changelog and VERSION
* git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
* git commit --amend
* # cherry pick the changes to the changelog
* git checkout ${BASE_TAG}-stable docs/content/changelog.md
* git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
* git push
* Announce!
## Making a manual build of docker

View File

@@ -1 +1 @@
v1.53.0
v1.53.1

View File

@@ -157,6 +157,17 @@ func driveScopesContainsAppFolder(scopes []string) bool {
return false
}
func driveOAuthOptions() []fs.Option {
opts := []fs.Option{}
for _, opt := range oauthutil.SharedOptions {
if opt.Name == config.ConfigClientID {
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
}
opts = append(opts, opt)
}
return opts
}
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
@@ -192,7 +203,7 @@ func init() {
log.Fatalf("Failed to configure team drive: %v", err)
}
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Options: append(driveOAuthOptions(), []fs.Option{{
Name: "scope",
Help: "Scope that rclone should use when requesting access from drive.",
Examples: []fs.OptionExample{{

View File

@@ -1213,7 +1213,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
// Set the file to be a sparse file (important on Windows)
err = file.SetSparse(out)
if err != nil {
fs.Debugf(o, "Failed to set sparse: %v", err)
fs.Errorf(o, "Failed to set sparse: %v", err)
}
}

View File

@@ -646,7 +646,6 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
400, // Bad request (seen in "Next token is expired")
401, // Unauthorized (seen in "Token has expired")
408, // Request Timeout
423, // Locked - get this on folders sometimes

View File

@@ -58,7 +58,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)",
NewFs: NewFs,
CommandHelp: commandHelp,
Options: []fs.Option{{
@@ -94,6 +94,9 @@ func init() {
}, {
Value: "StackPath",
Help: "StackPath Object Storage",
}, {
Value: "TencentCOS",
Help: "Tencent Cloud Object Storage (COS)",
}, {
Value: "Wasabi",
Help: "Wasabi Object Storage",
@@ -185,7 +188,7 @@ func init() {
}, {
Name: "region",
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba,Scaleway",
Provider: "!AWS,Alibaba,Scaleway,TencentCOS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -476,10 +479,73 @@ func init() {
Value: "s3.eu-central-1.stackpathstorage.com",
Help: "EU Endpoint",
}},
}, {
// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
Name: "endpoint",
Help: "Endpoint for Tencent COS API.",
Provider: "TencentCOS",
Examples: []fs.OptionExample{{
Value: "cos.ap-beijing.myqcloud.com",
Help: "Beijing Region.",
}, {
Value: "cos.ap-nanjing.myqcloud.com",
Help: "Nanjing Region.",
}, {
Value: "cos.ap-shanghai.myqcloud.com",
Help: "Shanghai Region.",
}, {
Value: "cos.ap-guangzhou.myqcloud.com",
Help: "Guangzhou Region.",
}, {
Value: "cos.ap-nanjing.myqcloud.com",
Help: "Nanjing Region.",
}, {
Value: "cos.ap-chengdu.myqcloud.com",
Help: "Chengdu Region.",
}, {
Value: "cos.ap-chongqing.myqcloud.com",
Help: "Chongqing Region.",
}, {
Value: "cos.ap-hongkong.myqcloud.com",
Help: "Hong Kong (China) Region.",
}, {
Value: "cos.ap-singapore.myqcloud.com",
Help: "Singapore Region.",
}, {
Value: "cos.ap-mumbai.myqcloud.com",
Help: "Mumbai Region.",
}, {
Value: "cos.ap-seoul.myqcloud.com",
Help: "Seoul Region.",
}, {
Value: "cos.ap-bangkok.myqcloud.com",
Help: "Bangkok Region.",
}, {
Value: "cos.ap-tokyo.myqcloud.com",
Help: "Tokyo Region.",
}, {
Value: "cos.na-siliconvalley.myqcloud.com",
Help: "Silicon Valley Region.",
}, {
Value: "cos.na-ashburn.myqcloud.com",
Help: "Virginia Region.",
}, {
Value: "cos.na-toronto.myqcloud.com",
Help: "Toronto Region.",
}, {
Value: "cos.eu-frankfurt.myqcloud.com",
Help: "Frankfurt Region.",
}, {
Value: "cos.eu-moscow.myqcloud.com",
Help: "Moscow Region.",
}, {
Value: "cos.accelerate.myqcloud.com",
Help: "Use Tencent COS Accelerate Endpoint.",
}},
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Help: "Dream Objects endpoint",
@@ -666,7 +732,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath,TencentCOS",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -678,9 +744,13 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.`,
Examples: []fs.OptionExample{{
Value: "default",
Help: "Owner gets Full_CONTROL. No one else has access rights (default).",
Provider: "TencentCOS",
}, {
Value: "private",
Help: "Owner gets FULL_CONTROL. No one else has access rights (default).",
Provider: "!IBMCOS",
Provider: "!IBMCOS,TencentCOS",
}, {
Value: "public-read",
Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
@@ -842,6 +912,24 @@ isn't set then "acl" is used instead.`,
Value: "STANDARD_IA",
Help: "Infrequent access storage mode.",
}},
}, {
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
Name: "storage_class",
Help: "The storage class to use when storing new objects in Tencent COS.",
Provider: "TencentCOS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Default",
}, {
Value: "STANDARD",
Help: "Standard storage class",
}, {
Value: "ARCHIVE",
Help: "Archive storage mode.",
}, {
Value: "STANDARD_IA",
Help: "Infrequent access storage mode.",
}},
}, {
// Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes
Name: "storage_class",
@@ -975,7 +1063,7 @@ if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to
false - rclone will do this automatically based on the provider
setting.`,
Default: true,
@@ -1305,7 +1393,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
if opt.Region == "" {
opt.Region = "us-east-1"
}
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint {
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.Provider == "TencentCOS" || opt.UseAccelerateEndpoint {
opt.ForcePathStyle = false
}
if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 {
@@ -1587,7 +1675,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
//
// So we enable only on providers we know supports it properly, all others can retry when a
// XML Syntax error is detected.
var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio")
var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio" || f.opt.Provider == "TencentCOS")
for {
// FIXME need to implement ALL loop
req := s3.ListObjectsInput{

View File

@@ -280,7 +280,7 @@ func stripVersion(goarch string) string {
// build the binary in dir returning success or failure
func compileArch(version, goos, goarch, dir string) bool {
log.Printf("Compiling %s/%s", goos, goarch)
log.Printf("Compiling %s/%s into %s", goos, goarch, dir)
output := filepath.Join(dir, "rclone")
if goos == "windows" {
output += ".exe"
@@ -298,7 +298,6 @@ func compileArch(version, goos, goarch, dir string) bool {
"go", "build",
"--ldflags", "-s -X github.com/rclone/rclone/fs.Version=" + version,
"-trimpath",
"-i",
"-o", output,
"-tags", *tags,
"..",
@@ -325,7 +324,7 @@ func compileArch(version, goos, goarch, dir string) bool {
artifacts := []string{buildZip(dir)}
// build a .deb and .rpm if appropriate
if goos == "linux" {
artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
artifacts = append(artifacts, buildDebAndRpm(dir, version, stripVersion(goarch))...)
}
if *copyAs != "" {
for _, artifact := range artifacts {

View File

@@ -29,6 +29,7 @@ var (
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by downloading rather than with hash.")
AddFlags(cmdFlags)
}
@@ -50,7 +51,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--src-only|
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--match|
and |--error| flags write paths, one per line, to the file name (or
stdout if it is |-|) supplied. What they write is described in the
help below. For example |--differ| will write all paths which are

View File

@@ -14,7 +14,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "cleanup remote:path",
Short: `Clean up the remote if possible`,
Short: `Clean up the remote if possible.`,
Long: `
Clean up the remote if possible. Empty the trash or delete old file
versions. Not supported by all remotes.

View File

@@ -22,7 +22,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "copy source:path dest:path",
Short: `Copy files from source to dest, skipping already copied`,
Short: `Copy files from source to dest, skipping already copied.`,
Long: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing by size and modification time or

View File

@@ -15,7 +15,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "copyto source:path dest:path",
Short: `Copy files from source to dest, skipping already copied`,
Short: `Copy files from source to dest, skipping already copied.`,
Long: `
If source:path is a file or directory then it copies it to a file or
directory named dest:path.

View File

@@ -44,7 +44,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "lsf remote:path",
Short: `List directories and objects in remote:path formatted for parsing`,
Short: `List directories and objects in remote:path formatted for parsing.`,
Long: `
List the contents of the source path (directories and objects) to
standard output in a form which is easy to parse by scripts. By

View File

@@ -192,6 +192,9 @@ Stopping the mount manually:
# OS X
umount /path/to/local/mount
**Note**: As of ` + "`rclone` 1.52.2, `rclone mount`" + ` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
### Installing on Windows
To run rclone ` + commandName + ` on Windows, you will need to

View File

@@ -17,7 +17,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "obscure password",
Short: `Obscure password for use in the rclone config file`,
Short: `Obscure password for use in the rclone config file.`,
Long: `In the rclone config file, human readable passwords are
obscured. Obscuring them is done by encrypting them and writing them
out in base64. This is **not** a secure way of encrypting these

View File

@@ -148,6 +148,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
{{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
{{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}}
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}

View File

@@ -5,6 +5,36 @@ description: "Rclone Changelog"
# Changelog
## v1.53.1 - 2020-09-13
[See commits](https://github.com/rclone/rclone/compare/v1.53.0...v1.53.1)
* Bug Fixes
* accounting: Remove new line from end of --stats-one-line display (Nick Craig-Wood)
* check
* Add back missing --download flag (Nick Craig-Wood)
* Fix docs (Nick Craig-Wood)
* docs
* Note --log-file does append (Nick Craig-Wood)
* Add full stops for consistency in rclone --help (edwardxml)
* Add Tencent COS to s3 provider list (wjielai)
* Updated mount command to reflect that it requires Go 1.13 or newer (Evan Harris)
* jottacloud: Mention that uploads from local disk will not need to cache files to disk for md5 calculation (albertony)
* Fix formatting of rc docs page (Nick Craig-Wood)
* build
* Include vendor tar ball in release and fix startdev (Nick Craig-Wood)
* Fix "Illegal instruction" error for ARMv6 builds (Nick Craig-Wood)
* Fix architecture name in ARMv7 build (Nick Craig-Wood)
* VFS
* Fix spurious error "vfs cache: failed to _ensure cache EOF" (Nick Craig-Wood)
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Local
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Drive
* Re-adds special oauth help text (Tim Gallant)
* Opendrive
* Do not retry 400 errors (Evan Harris)
## v1.53.0 - 2020-09-02
[See commits](https://github.com/rclone/rclone/compare/v1.52.0...v1.53.0)

View File

@@ -39,10 +39,10 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone backend](/commands/rclone_backend/) - Run a backend specific command.
* [rclone cat](/commands/rclone_cat/) - Concatenates any files and sends them to stdout.
* [rclone check](/commands/rclone_check/) - Checks the files in the source and destination match.
* [rclone cleanup](/commands/rclone_cleanup/) - Clean up the remote if possible
* [rclone cleanup](/commands/rclone_cleanup/) - Clean up the remote if possible.
* [rclone config](/commands/rclone_config/) - Enter an interactive configuration session.
* [rclone copy](/commands/rclone_copy/) - Copy files from source to dest, skipping already copied
* [rclone copyto](/commands/rclone_copyto/) - Copy files from source to dest, skipping already copied
* [rclone copy](/commands/rclone_copy/) - Copy files from source to dest, skipping already copied.
* [rclone copyto](/commands/rclone_copyto/) - Copy files from source to dest, skipping already copied.
* [rclone copyurl](/commands/rclone_copyurl/) - Copy url content to dest.
* [rclone cryptcheck](/commands/rclone_cryptcheck/) - Cryptcheck checks the integrity of a crypted remote.
* [rclone cryptdecode](/commands/rclone_cryptdecode/) - Cryptdecode returns unencrypted file names.
@@ -56,7 +56,7 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone listremotes](/commands/rclone_listremotes/) - List all the remotes in the config file.
* [rclone ls](/commands/rclone_ls/) - List the objects in the path with size and path.
* [rclone lsd](/commands/rclone_lsd/) - List all directories/containers/buckets in the path.
* [rclone lsf](/commands/rclone_lsf/) - List directories and objects in remote:path formatted for parsing
* [rclone lsf](/commands/rclone_lsf/) - List directories and objects in remote:path formatted for parsing.
* [rclone lsjson](/commands/rclone_lsjson/) - List directories and objects in the path in JSON format.
* [rclone lsl](/commands/rclone_lsl/) - List the objects in path with modification time, size and path.
* [rclone md5sum](/commands/rclone_md5sum/) - Produces an md5sum file for all the objects in the path.
@@ -65,7 +65,7 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone move](/commands/rclone_move/) - Move files from source to dest.
* [rclone moveto](/commands/rclone_moveto/) - Move file or directory from source to dest.
* [rclone ncdu](/commands/rclone_ncdu/) - Explore a remote with a text based user interface.
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone config file
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone config file.
* [rclone purge](/commands/rclone_purge/) - Remove the path and all of its contents.
* [rclone rc](/commands/rclone_rc/) - Run a command against a running rclone.
* [rclone rcat](/commands/rclone_rcat/) - Copies standard input to file on remote.

View File

@@ -29,7 +29,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
@@ -55,6 +55,7 @@ rclone check source:path dest:path [flags]
```
--combined string Make a combined report of changes to this file
--differ string Report all non-matching files to this file
--download Check by downloading rather than with hash.
--error string Report all files with errors (hashing or reading) to this file
-h, --help help for check
--match string Report all matching files to this file

View File

@@ -1,13 +1,13 @@
---
title: "rclone cleanup"
description: "Clean up the remote if possible"
description: "Clean up the remote if possible."
slug: rclone_cleanup
url: /commands/rclone_cleanup/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/cleanup/ and as part of making a release run "make commanddocs"
---
# rclone cleanup
Clean up the remote if possible
Clean up the remote if possible.
## Synopsis

View File

@@ -1,13 +1,13 @@
---
title: "rclone copy"
description: "Copy files from source to dest, skipping already copied"
description: "Copy files from source to dest, skipping already copied."
slug: rclone_copy
url: /commands/rclone_copy/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/copy/ and as part of making a release run "make commanddocs"
---
# rclone copy
Copy files from source to dest, skipping already copied
Copy files from source to dest, skipping already copied.
## Synopsis

View File

@@ -1,13 +1,13 @@
---
title: "rclone copyto"
description: "Copy files from source to dest, skipping already copied"
description: "Copy files from source to dest, skipping already copied."
slug: rclone_copyto
url: /commands/rclone_copyto/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/copyto/ and as part of making a release run "make commanddocs"
---
# rclone copyto
Copy files from source to dest, skipping already copied
Copy files from source to dest, skipping already copied.
## Synopsis

View File

@@ -40,7 +40,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are

View File

@@ -1,13 +1,13 @@
---
title: "rclone lsf"
description: "List directories and objects in remote:path formatted for parsing"
description: "List directories and objects in remote:path formatted for parsing."
slug: rclone_lsf
url: /commands/rclone_lsf/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/lsf/ and as part of making a release run "make commanddocs"
---
# rclone lsf
List directories and objects in remote:path formatted for parsing
List directories and objects in remote:path formatted for parsing.
## Synopsis

View File

@@ -49,6 +49,9 @@ Stopping the mount manually:
# OS X
umount /path/to/local/mount
**Note**: As of `rclone` 1.52.2, `rclone mount` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
## Installing on Windows
To run rclone mount on Windows, you will need to
@@ -357,6 +360,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -1,13 +1,13 @@
---
title: "rclone obscure"
description: "Obscure password for use in the rclone config file"
description: "Obscure password for use in the rclone config file."
slug: rclone_obscure
url: /commands/rclone_obscure/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/obscure/ and as part of making a release run "make commanddocs"
---
# rclone obscure
Obscure password for use in the rclone config file
Obscure password for use in the rclone config file.
## Synopsis

View File

@@ -196,6 +196,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -195,6 +195,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -267,6 +267,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -206,6 +206,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -275,6 +275,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -757,6 +757,8 @@ This can be useful for tracking down problems with syncs in
combination with the `-v` flag. See the [Logging section](#logging)
for more info.
If FILE exists then rclone will append to it.
Note that if you are using the `logrotate` program to manage rclone's
logs, then you should use the `copytruncate` option as rclone doesn't
have a signal to rotate logs.

View File

@@ -547,8 +547,10 @@ Here are the standard options specific to drive (Google Drive).
#### --drive-client-id
OAuth Client Id
Leave blank normally.
Google Application Client Id
Setting your own is recommended.
See https://rclone.org/drive/#making-your-own-client-id for how to create your own.
If you leave this blank, it will use an internal key which is low performance.
- Config: client_id
- Env Var: RCLONE_DRIVE_CLIENT_ID

View File

@@ -147,7 +147,7 @@ These flags are available for every command.
--use-json-log Use json log format.
--use-mmap Use mmap allocator (see docs).
--use-server-modtime Use server modified time instead of object metadata
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.0")
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.1")
-v, --verbose count Print lots more stuff (repeat for more)
```
@@ -246,7 +246,7 @@ and may be set in the config file.
--drive-auth-owner-only Only consider files owned by the authenticated user.
--drive-auth-url string Auth server URL.
--drive-chunk-size SizeSuffix Upload chunk size. Must a power of 2 >= 256k. (default 8M)
--drive-client-id string OAuth Client Id
--drive-client-id string Google Application Client Id
--drive-client-secret string OAuth Client Secret
--drive-disable-http2 Disable drive using http2 (default true)
--drive-encoding MultiEncoder This sets the encoding for the backend. (default InvalidUtf8)

View File

@@ -148,8 +148,13 @@ flag.
Note that Jottacloud requires the MD5 hash before upload so if the
source does not have an MD5 checksum then the file will be cached
temporarily on disk (wherever the `TMPDIR` environment variable points
to) before it is uploaded. Small files will be cached in memory - see
to) before it is uploaded. Small files will be cached in memory - see
the [--jottacloud-md5-memory-limit](#jottacloud-md5-memory-limit) flag.
When uploading from local disk the source checksum is always available,
so this does not apply. Starting with rclone version 1.52 the same is
true for crypted remotes (in older versions the crypt backend would not
calculate hashes for uploads from local disk, so the Jottacloud
backend had to do it as described above).
#### Restricted filename characters

View File

@@ -537,6 +537,8 @@ OR
"result": "<Raw command line output>"
}
```
**Authentication is required for this call.**
### core/gc: Runs a garbage collection. {#core-gc}
@@ -1212,7 +1214,7 @@ This allows you to remove a plugin using it's name
This takes parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format `author`/`plugin_name`
Eg
@@ -1226,7 +1228,7 @@ This allows you to remove a plugin using it's name
This takes the following parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format `author`/`plugin_name`
Eg

View File

@@ -18,6 +18,7 @@ The S3 backend can be used with a number of different providers:
{{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
{{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
{{< /provider_list >}}
@@ -455,7 +456,7 @@ Vault API, so rclone cannot directly access Glacier Vaults.
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
### Standard Options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
#### --s3-provider
@@ -486,6 +487,8 @@ Choose your S3 provider.
- Scaleway Object Storage
- "StackPath"
- StackPath Object Storage
- "TencentCOS"
- Tencent Cloud Object Storage (COS)
- "Wasabi"
- Wasabi Object Storage
- "Other"
@@ -839,6 +842,54 @@ Endpoint for StackPath Object Storage.
#### --s3-endpoint
Endpoint for Tencent COS API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Type: string
- Default: ""
- Examples:
- "cos.ap-beijing.myqcloud.com"
- Beijing Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-shanghai.myqcloud.com"
- Shanghai Region.
- "cos.ap-guangzhou.myqcloud.com"
- Guangzhou Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-chengdu.myqcloud.com"
- Chengdu Region.
- "cos.ap-chongqing.myqcloud.com"
- Chongqing Region.
- "cos.ap-hongkong.myqcloud.com"
- Hong Kong (China) Region.
- "cos.ap-singapore.myqcloud.com"
- Singapore Region.
- "cos.ap-mumbai.myqcloud.com"
- Mumbai Region.
- "cos.ap-seoul.myqcloud.com"
- Seoul Region.
- "cos.ap-bangkok.myqcloud.com"
- Bangkok Region.
- "cos.ap-tokyo.myqcloud.com"
- Tokyo Region.
- "cos.na-siliconvalley.myqcloud.com"
- Silicon Valley Region.
- "cos.na-ashburn.myqcloud.com"
- Virginia Region.
- "cos.na-toronto.myqcloud.com"
- Toronto Region.
- "cos.eu-frankfurt.myqcloud.com"
- Frankfurt Region.
- "cos.eu-moscow.myqcloud.com"
- Moscow Region.
- "cos.accelerate.myqcloud.com"
- Use Tencent COS Accelerate Endpoint.
#### --s3-endpoint
Endpoint for S3 API.
Required when using an S3 clone.
@@ -1006,6 +1057,8 @@ doesn't copy the ACL from the source but rather writes a fresh one.
- Type: string
- Default: ""
- Examples:
- "default"
- Owner gets Full_CONTROL. No one else has access rights (default).
- "private"
- Owner gets FULL_CONTROL. No one else has access rights (default).
- "public-read"
@@ -1106,6 +1159,24 @@ The storage class to use when storing new objects in OSS.
#### --s3-storage-class
The storage class to use when storing new objects in Tencent COS.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Type: string
- Default: ""
- Examples:
- ""
- Default
- "STANDARD"
- Standard storage class
- "ARCHIVE"
- Archive storage mode.
- "STANDARD_IA"
- Infrequent access storage mode.
#### --s3-storage-class
The storage class to use when storing new objects in S3.
- Config: storage_class
@@ -1122,7 +1193,7 @@ The storage class to use when storing new objects in S3.
### Advanced Options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
#### --s3-bucket-acl
@@ -1343,7 +1414,7 @@ if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to
false - rclone will do this automatically based on the provider
setting.
@@ -2212,6 +2283,138 @@ d) Delete this remote
y/e/d> y
```
### Tencent COS {#tencent-cos}
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
To configure access to Tencent COS, follow the steps below:
1. Run `rclone config` and select `n` for a new remote.
```
rclone config
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
```
2. Give the name of the configuration. For example, name it 'cos'.
```
name> cos
```
3. Select `s3` storage.
```
Choose a number from below, or type in your own value
1 / 1Fichier
\ "fichier"
2 / Alias for an existing remote
\ "alias"
3 / Amazon Drive
\ "amazon cloud drive"
4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)
\ "s3"
[snip]
Storage> s3
```
4. Select `TencentCOS` provider.
```
Choose a number from below, or type in your own value
1 / Amazon Web Services (AWS) S3
\ "AWS"
[snip]
11 / Tencent Cloud Object Storage (COS)
\ "TencentCOS"
[snip]
provider> TencentCOS
```
5. Enter your SecretId and SecretKey of Tencent Cloud.
```
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Enter a boolean value (true or false). Press Enter for the default ("false").
Choose a number from below, or type in your own value
1 / Enter AWS credentials in the next step
\ "false"
2 / Get AWS credentials from the environment (env vars or IAM)
\ "true"
env_auth> 1
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
access_key_id> AKIDxxxxxxxxxx
AWS Secret Access Key (password)
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
secret_access_key> xxxxxxxxxxx
```
6. Select endpoint for Tencent COS. This is the standard endpoint for different region.
```
1 / Beijing Region.
\ "cos.ap-beijing.myqcloud.com"
2 / Nanjing Region.
\ "cos.ap-nanjing.myqcloud.com"
3 / Shanghai Region.
\ "cos.ap-shanghai.myqcloud.com"
4 / Guangzhou Region.
\ "cos.ap-guangzhou.myqcloud.com"
[snip]
endpoint> 4
```
7. Choose acl and storage class.
```
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Owner gets Full_CONTROL. No one else has access rights (default).
\ "default"
[snip]
acl> 1
The storage class to use when storing new objects in Tencent COS.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Default
\ ""
[snip]
storage_class> 1
Edit advanced config? (y/n)
y) Yes
n) No (default)
y/n> n
Remote config
--------------------
[cos]
type = s3
provider = TencentCOS
env_auth = false
access_key_id = xxx
secret_access_key = xxx
endpoint = cos.ap-guangzhou.myqcloud.com
acl = default
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
Current remotes:
Name Type
==== ====
cos s3
```
### Netease NOS ###
For Netease NOS configure as per the configurator `rclone config`

View File

@@ -1 +1 @@
v1.53.0
v1.53.1

View File

@@ -272,7 +272,7 @@ func (s *StatsInfo) String() string {
}
}
_, _ = fmt.Fprintf(buf, "%s%10s / %s, %s, %s, ETA %s%s\n",
_, _ = fmt.Fprintf(buf, "%s%10s / %s, %s, %s, ETA %s%s",
dateString,
fs.SizeSuffix(s.bytes),
fs.SizeSuffix(totalSize).Unit("Bytes"),
@@ -283,6 +283,7 @@ func (s *StatsInfo) String() string {
)
if !fs.Config.StatsOneLine {
_, _ = buf.WriteRune('\n')
errorDetails := ""
switch {
case s.fatalError:
@@ -291,6 +292,7 @@ func (s *StatsInfo) String() string {
errorDetails = " (retrying may help)"
case s.errors != 0:
errorDetails = " (no need to retry)"
}
// Add only non zero stats

View File

@@ -379,7 +379,7 @@ OR
"result": "<Raw command line output>"
}
` + "```" + `
`,
})
}

View File

@@ -45,7 +45,7 @@ func init() {
This takes the following parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format ` + "`author`/`plugin_name`" + `
Eg
@@ -212,7 +212,7 @@ func init() {
This takes parameters
- name: name of the plugin in the format <author>/<plugin_name>
- name: name of the plugin in the format ` + "`author`/`plugin_name`" + `
Eg

View File

@@ -1,4 +1,4 @@
package fs
// Version of rclone
var Version = "v1.53.0-DEV"
var Version = "v1.53.1-DEV"

23830
rclone.1 generated

File diff suppressed because it is too large Load Diff

View File

@@ -166,6 +166,11 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
### VFS Performance
These flags may be used to enable/disable features of the VFS for

View File

@@ -249,7 +249,7 @@ func (item *Item) _truncate(size int64) (err error) {
err = file.SetSparse(fd)
if err != nil {
fs.Debugf(item.name, "vfs cache: truncate: failed to set as a sparse file: %v", err)
fs.Errorf(item.name, "vfs cache: truncate: failed to set as a sparse file: %v", err)
}
}
@@ -446,7 +446,7 @@ func (item *Item) _createFile(osPath string) (err error) {
}
err = file.SetSparse(fd)
if err != nil {
fs.Debugf(item.name, "vfs cache: failed to set as a sparse file: %v", err)
fs.Errorf(item.name, "vfs cache: failed to set as a sparse file: %v", err)
}
item.fd = fd
@@ -1158,7 +1158,7 @@ func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
item.preAccess()
n, err = item.readAt(b, off)
item.postAccess()
if err == nil {
if err == nil || err == io.EOF {
break
}
fs.Errorf(item.name, "vfs cache: failed to _ensure cache %v", err)