mirror of
https://github.com/rclone/rclone.git
synced 2026-01-21 20:03:22 +00:00
Compare commits
1 Commits
v1.55.0
...
zoho-clien
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
61316f4ef6 |
130
.github/workflows/build.yml
vendored
130
.github/workflows/build.yml
vendored
@@ -213,98 +213,50 @@ jobs:
|
||||
# Deploy binaries if enabled in config && not a PR && not a fork
|
||||
if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
android:
|
||||
timeout-minutes: 30
|
||||
name: "android-all"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
xgo:
|
||||
timeout-minutes: 60
|
||||
name: "xgo cross compile"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go 1.14
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.14
|
||||
steps:
|
||||
|
||||
# Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
|
||||
- name: Force NDK version
|
||||
run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;21.4.7075529" | grep -v = || true
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v1
|
||||
with:
|
||||
# Checkout into a fixed path to avoid import path problems on go < 1.11
|
||||
path: ./src/github.com/rclone/rclone
|
||||
|
||||
- name: Go module cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
- name: Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'GOPATH=${{ runner.workspace }}' >> $GITHUB_ENV
|
||||
echo '${{ runner.workspace }}/bin' >> $GITHUB_PATH
|
||||
|
||||
- name: Set global environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "VERSION=$(make version)" >> $GITHUB_ENV
|
||||
- name: Cross-compile rclone
|
||||
run: |
|
||||
docker pull billziss/xgo-cgofuse
|
||||
GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
|
||||
# xgo \
|
||||
# -image=billziss/xgo-cgofuse \
|
||||
# -targets=darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
|
||||
# -tags cmount \
|
||||
# -dest build \
|
||||
# .
|
||||
xgo \
|
||||
-image=billziss/xgo-cgofuse \
|
||||
-targets=android/*,ios/* \
|
||||
-dest build \
|
||||
.
|
||||
|
||||
- name: build native rclone
|
||||
run: |
|
||||
make
|
||||
- name: Build rclone
|
||||
shell: bash
|
||||
run: |
|
||||
make
|
||||
|
||||
- name: arm-v7a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=arm' >> $GITHUB_ENV
|
||||
echo 'GOARM=7' >> $GITHUB_ENV
|
||||
echo 'CGO_ENABLED=1' >> $GITHUB_ENV
|
||||
echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
|
||||
- name: arm-v7a build
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a .
|
||||
|
||||
- name: arm64-v8a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=arm64' >> $GITHUB_ENV
|
||||
echo 'CGO_ENABLED=1' >> $GITHUB_ENV
|
||||
echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
|
||||
|
||||
- name: arm64-v8a build
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a .
|
||||
|
||||
- name: x86 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=386' >> $GITHUB_ENV
|
||||
echo 'CGO_ENABLED=1' >> $GITHUB_ENV
|
||||
echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
|
||||
|
||||
- name: x86 build
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 .
|
||||
|
||||
- name: x64 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=amd64' >> $GITHUB_ENV
|
||||
echo 'CGO_ENABLED=1' >> $GITHUB_ENV
|
||||
echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
|
||||
|
||||
- name: x64 build
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 .
|
||||
|
||||
- name: Upload artifacts
|
||||
run: |
|
||||
make ci_upload
|
||||
env:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
- name: Upload artifacts
|
||||
run: |
|
||||
make ci_upload
|
||||
env:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,7 +1,6 @@
|
||||
*~
|
||||
_junk/
|
||||
rclone
|
||||
rclone.exe
|
||||
build
|
||||
docs/public
|
||||
rclone.iml
|
||||
|
||||
1289
MANUAL.html
generated
1289
MANUAL.html
generated
File diff suppressed because it is too large
Load Diff
1867
MANUAL.txt
generated
1867
MANUAL.txt
generated
File diff suppressed because it is too large
Load Diff
8
Makefile
8
Makefile
@@ -119,7 +119,7 @@ doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
|
||||
rclone.1: MANUAL.md
|
||||
pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
|
||||
|
||||
MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
|
||||
MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs backenddocs
|
||||
./bin/make_manual.py
|
||||
|
||||
MANUAL.html: MANUAL.md
|
||||
@@ -187,10 +187,10 @@ upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
|
||||
cross: doc
|
||||
go run bin/cross-compile.go -release current $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
go run bin/cross-compile.go -release current $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
|
||||
beta:
|
||||
go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
go run bin/cross-compile.go $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
|
||||
@echo Beta release ready at https://pub.rclone.org/$(TAG)/
|
||||
|
||||
@@ -198,7 +198,7 @@ log_since_last_release:
|
||||
git log $(LAST_TAG)..
|
||||
|
||||
compile_all:
|
||||
go run bin/cross-compile.go -compile-only $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
|
||||
|
||||
ci_upload:
|
||||
sudo chown -R $$USER build
|
||||
|
||||
18
RELEASE.md
18
RELEASE.md
@@ -76,24 +76,6 @@ Now
|
||||
The rclone docker image should autobuild on via GitHub actions. If it doesn't
|
||||
or needs to be updated then rebuild like this.
|
||||
|
||||
See: https://github.com/ilteoood/docker_buildx/issues/19
|
||||
See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
|
||||
|
||||
```
|
||||
git co v1.54.1
|
||||
docker pull golang
|
||||
export DOCKER_CLI_EXPERIMENTAL=enabled
|
||||
docker buildx create --name actions_builder --use
|
||||
docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
|
||||
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
||||
SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
|
||||
echo "Supported platforms: $SUPPORTED_PLATFORMS"
|
||||
docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
|
||||
docker buildx stop actions_builder
|
||||
```
|
||||
|
||||
### Old build for linux/amd64 only
|
||||
|
||||
```
|
||||
docker pull golang
|
||||
docker build --rm --ulimit memlock=67108864 -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/local" // pull in test backend
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -20,7 +19,7 @@ var (
|
||||
)
|
||||
|
||||
func prepare(t *testing.T, root string) {
|
||||
configfile.LoadConfig(context.Background())
|
||||
config.LoadConfig(context.Background())
|
||||
|
||||
// Configure the remote
|
||||
config.FileSet(remoteName, "type", "alias")
|
||||
|
||||
@@ -205,10 +205,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 401 {
|
||||
f.tokenRenewer.Invalidate()
|
||||
@@ -283,7 +280,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
// Renew the token in the background
|
||||
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
|
||||
_, err := f.getRootInfo(ctx)
|
||||
_, err := f.getRootInfo()
|
||||
return err
|
||||
})
|
||||
|
||||
@@ -291,14 +288,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, resp, err = f.c.Account.GetEndpoints()
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get endpoints")
|
||||
}
|
||||
|
||||
// Get rootID
|
||||
rootInfo, err := f.getRootInfo(ctx)
|
||||
rootInfo, err := f.getRootInfo()
|
||||
if err != nil || rootInfo.Id == nil {
|
||||
return nil, errors.Wrap(err, "failed to get root")
|
||||
}
|
||||
@@ -340,11 +337,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// getRootInfo gets the root folder info
|
||||
func (f *Fs) getRootInfo(ctx context.Context) (rootInfo *acd.Folder, err error) {
|
||||
func (f *Fs) getRootInfo() (rootInfo *acd.Folder, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
rootInfo, resp, err = f.c.Nodes.GetRoot()
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
return rootInfo, err
|
||||
}
|
||||
@@ -383,7 +380,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
var subFolder *acd.Folder
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if err == acd.ErrorNodeNotFound {
|
||||
@@ -410,7 +407,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
var info *acd.Folder
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -431,7 +428,7 @@ type listAllFn func(*acd.Node) bool
|
||||
// Lists the directory required calling the user function on each item found
|
||||
//
|
||||
// If the user fn ever returns true then it early exits with found = true
|
||||
func (f *Fs) listAll(ctx context.Context, dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
|
||||
func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
|
||||
query := "parents:" + dirID
|
||||
if directoriesOnly {
|
||||
query += " AND kind:" + folderKind
|
||||
@@ -452,7 +449,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, title string, directorie
|
||||
var resp *http.Response
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
nodes, resp, err = f.c.Nodes.GetNodes(&opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -511,7 +508,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
var iErr error
|
||||
for tries := 1; tries <= maxTries; tries++ {
|
||||
entries = nil
|
||||
_, err = f.listAll(ctx, directoryID, "", false, false, func(node *acd.Node) bool {
|
||||
_, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool {
|
||||
remote := path.Join(dir, *node.Name)
|
||||
switch *node.Kind {
|
||||
case folderKind:
|
||||
@@ -670,7 +667,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if ok {
|
||||
return false, nil
|
||||
}
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -711,7 +708,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = f.moveNode(ctx, srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
|
||||
err = f.moveNode(srcObj.remote, dstLeaf, dstDirectoryID, srcObj.info, srcLeaf, srcDirectoryID, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -806,7 +803,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
var jsonStr string
|
||||
err = srcFs.pacer.Call(func() (bool, error) {
|
||||
jsonStr, err = srcInfo.GetMetadata()
|
||||
return srcFs.shouldRetry(ctx, nil, err)
|
||||
return srcFs.shouldRetry(nil, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(src, "DirMove error: error reading src metadata: %v", err)
|
||||
@@ -818,7 +815,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return err
|
||||
}
|
||||
|
||||
err = f.moveNode(ctx, srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
|
||||
err = f.moveNode(srcPath, dstLeaf, dstDirectoryID, srcInfo, srcLeaf, srcDirectoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -843,7 +840,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
if check {
|
||||
// check directory is empty
|
||||
empty := true
|
||||
_, err = f.listAll(ctx, rootID, "", false, false, func(node *acd.Node) bool {
|
||||
_, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool {
|
||||
switch *node.Kind {
|
||||
case folderKind:
|
||||
empty = false
|
||||
@@ -868,7 +865,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = node.Trash()
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -990,7 +987,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
var info *acd.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if err == acd.ErrorNodeNotFound {
|
||||
@@ -1047,7 +1044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
} else {
|
||||
in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
|
||||
}
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
return in, err
|
||||
}
|
||||
@@ -1070,7 +1067,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if ok {
|
||||
return false, nil
|
||||
}
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1080,70 +1077,70 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
|
||||
// Remove a node
|
||||
func (f *Fs) removeNode(ctx context.Context, info *acd.Node) error {
|
||||
func (f *Fs) removeNode(info *acd.Node) error {
|
||||
var resp *http.Response
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = info.Trash()
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.removeNode(ctx, o.info)
|
||||
return o.fs.removeNode(o.info)
|
||||
}
|
||||
|
||||
// Restore a node
|
||||
func (f *Fs) restoreNode(ctx context.Context, info *acd.Node) (newInfo *acd.Node, err error) {
|
||||
func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
newInfo, resp, err = info.Restore()
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
return newInfo, err
|
||||
}
|
||||
|
||||
// Changes name of given node
|
||||
func (f *Fs) renameNode(ctx context.Context, info *acd.Node, newName string) (newInfo *acd.Node, err error) {
|
||||
func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
return newInfo, err
|
||||
}
|
||||
|
||||
// Replaces one parent with another, effectively moving the file. Leaves other
|
||||
// parents untouched. ReplaceParent cannot be used when the file is trashed.
|
||||
func (f *Fs) replaceParent(ctx context.Context, info *acd.Node, oldParentID string, newParentID string) error {
|
||||
func (f *Fs) replaceParent(info *acd.Node, oldParentID string, newParentID string) error {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := info.ReplaceParent(oldParentID, newParentID)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
// Adds one additional parent to object.
|
||||
func (f *Fs) addParent(ctx context.Context, info *acd.Node, newParentID string) error {
|
||||
func (f *Fs) addParent(info *acd.Node, newParentID string) error {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := info.AddParent(newParentID)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
// Remove given parent from object, leaving the other possible
|
||||
// parents untouched. Object can end up having no parents.
|
||||
func (f *Fs) removeParent(ctx context.Context, info *acd.Node, parentID string) error {
|
||||
func (f *Fs) removeParent(info *acd.Node, parentID string) error {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := info.RemoveParent(parentID)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
// moveNode moves the node given from the srcLeaf,srcDirectoryID to
|
||||
// the dstLeaf,dstDirectoryID
|
||||
func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
|
||||
func (f *Fs) moveNode(name, dstLeaf, dstDirectoryID string, srcInfo *acd.Node, srcLeaf, srcDirectoryID string, useDirErrorMsgs bool) (err error) {
|
||||
// fs.Debugf(name, "moveNode dst(%q,%s) <- src(%q,%s)", dstLeaf, dstDirectoryID, srcLeaf, srcDirectoryID)
|
||||
cantMove := fs.ErrorCantMove
|
||||
if useDirErrorMsgs {
|
||||
@@ -1157,7 +1154,7 @@ func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string,
|
||||
|
||||
if srcLeaf != dstLeaf {
|
||||
// fs.Debugf(name, "renaming")
|
||||
_, err = f.renameNode(ctx, srcInfo, dstLeaf)
|
||||
_, err = f.renameNode(srcInfo, dstLeaf)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: quick path rename failed: %v", err)
|
||||
goto OnConflict
|
||||
@@ -1165,7 +1162,7 @@ func (f *Fs) moveNode(ctx context.Context, name, dstLeaf, dstDirectoryID string,
|
||||
}
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
// fs.Debugf(name, "trying parent replace: %s -> %s", oldParentID, newParentID)
|
||||
err = f.replaceParent(ctx, srcInfo, srcDirectoryID, dstDirectoryID)
|
||||
err = f.replaceParent(srcInfo, srcDirectoryID, dstDirectoryID)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: quick path parent replace failed: %v", err)
|
||||
return err
|
||||
@@ -1178,13 +1175,13 @@ OnConflict:
|
||||
fs.Debugf(name, "Could not directly rename file, presumably because there was a file with the same name already. Instead, the file will now be trashed where such operations do not cause errors. It will be restored to the correct parent after. If any of the subsequent calls fails, the rename/move will be in an invalid state.")
|
||||
|
||||
// fs.Debugf(name, "Trashing file")
|
||||
err = f.removeNode(ctx, srcInfo)
|
||||
err = f.removeNode(srcInfo)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: remove node failed: %v", err)
|
||||
return err
|
||||
}
|
||||
// fs.Debugf(name, "Renaming file")
|
||||
_, err = f.renameNode(ctx, srcInfo, dstLeaf)
|
||||
_, err = f.renameNode(srcInfo, dstLeaf)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: rename node failed: %v", err)
|
||||
return err
|
||||
@@ -1192,19 +1189,19 @@ OnConflict:
|
||||
// note: replacing parent is forbidden by API, modifying them individually is
|
||||
// okay though
|
||||
// fs.Debugf(name, "Adding target parent")
|
||||
err = f.addParent(ctx, srcInfo, dstDirectoryID)
|
||||
err = f.addParent(srcInfo, dstDirectoryID)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: addParent failed: %v", err)
|
||||
return err
|
||||
}
|
||||
// fs.Debugf(name, "removing original parent")
|
||||
err = f.removeParent(ctx, srcInfo, srcDirectoryID)
|
||||
err = f.removeParent(srcInfo, srcDirectoryID)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: removeParent failed: %v", err)
|
||||
return err
|
||||
}
|
||||
// fs.Debugf(name, "Restoring")
|
||||
_, err = f.restoreNode(ctx, srcInfo)
|
||||
_, err = f.restoreNode(srcInfo)
|
||||
if err != nil {
|
||||
fs.Debugf(name, "Move: restoreNode node failed: %v", err)
|
||||
return err
|
||||
|
||||
@@ -217,23 +217,6 @@ This option controls how often unused buffers will be removed from the pool.`,
|
||||
encoder.EncodeDel |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeRightPeriod),
|
||||
}, {
|
||||
Name: "public_access",
|
||||
Help: "Public access level of a container: blob, container.",
|
||||
Default: string(azblob.PublicAccessNone),
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: string(azblob.PublicAccessNone),
|
||||
Help: "The container and its blobs can be accessed only with an authorized request. It's a default value",
|
||||
}, {
|
||||
Value: string(azblob.PublicAccessBlob),
|
||||
Help: "Blob data within this container can be read via anonymous request.",
|
||||
}, {
|
||||
Value: string(azblob.PublicAccessContainer),
|
||||
Help: "Allow full public read access for container and blob data.",
|
||||
},
|
||||
},
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -258,7 +241,6 @@ type Options struct {
|
||||
MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"`
|
||||
MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
PublicAccess string `config:"public_access"`
|
||||
}
|
||||
|
||||
// Fs represents a remote azure server
|
||||
@@ -280,7 +262,6 @@ type Fs struct {
|
||||
imdsPacer *fs.Pacer // Same but for IMDS
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
pool *pool.Pool // memory pool
|
||||
publicAccess azblob.PublicAccessType // Container Public Access Level
|
||||
}
|
||||
|
||||
// Object describes an azure object
|
||||
@@ -354,19 +335,6 @@ func validateAccessTier(tier string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// validatePublicAccess checks if azureblob supports use supplied public access level
|
||||
func validatePublicAccess(publicAccess string) bool {
|
||||
switch publicAccess {
|
||||
case string(azblob.PublicAccessNone),
|
||||
string(azblob.PublicAccessBlob),
|
||||
string(azblob.PublicAccessContainer):
|
||||
// valid cases
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
401, // Unauthorized (e.g. "Token has expired")
|
||||
@@ -379,10 +347,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(err error) (bool, error) {
|
||||
// FIXME interpret special errors - more to do here
|
||||
if storageErr, ok := err.(azblob.StorageError); ok {
|
||||
switch storageErr.ServiceCode() {
|
||||
@@ -534,11 +499,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
|
||||
}
|
||||
|
||||
if !validatePublicAccess((opt.PublicAccess)) {
|
||||
return nil, errors.Errorf("Azure Blob: Supported public access level are %s and %s",
|
||||
string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer))
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
@@ -557,7 +517,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
opt.MemoryPoolUseMmap,
|
||||
),
|
||||
}
|
||||
f.publicAccess = azblob.PublicAccessType(opt.PublicAccess)
|
||||
f.imdsPacer.SetRetries(5) // per IMDS documentation
|
||||
f.setRoot(root)
|
||||
f.features = (&fs.Features{
|
||||
@@ -619,7 +578,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// Retry as specified by the documentation:
|
||||
// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
|
||||
token, err = GetMSIToken(ctx, userMSI)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -635,7 +594,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
var refreshedToken adal.Token
|
||||
err := f.imdsPacer.Call(func() (bool, error) {
|
||||
refreshedToken, err = GetMSIToken(ctx, userMSI)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
// Failed to refresh.
|
||||
@@ -844,7 +803,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -1070,7 +1029,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1122,7 +1081,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
|
||||
}
|
||||
// now try to create the container
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, f.publicAccess)
|
||||
_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
|
||||
if err != nil {
|
||||
if storageErr, ok := err.(azblob.StorageError); ok {
|
||||
switch storageErr.ServiceCode() {
|
||||
@@ -1139,7 +1098,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
}, nil)
|
||||
}
|
||||
@@ -1177,10 +1136,10 @@ func (f *Fs) deleteContainer(ctx context.Context, container string) error {
|
||||
return false, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
}
|
||||
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -1253,7 +1212,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1414,7 +1373,7 @@ func (o *Object) readMetaData() (err error) {
|
||||
var blobProperties *azblob.BlobGetPropertiesResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
|
||||
@@ -1449,7 +1408,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
blob := o.getBlobReference()
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1492,7 +1451,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var downloadResponse *azblob.DownloadResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open for download")
|
||||
@@ -1633,7 +1592,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Stream contents of the reader object to the given blob URL
|
||||
blockBlobURL := blob.ToBlockBlobURL()
|
||||
_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1661,7 +1620,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
ac := azblob.BlobAccessConditions{}
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := blob.Delete(ctx, snapShotOptions, ac)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1690,7 +1649,7 @@ func (o *Object) SetTier(tier string) error {
|
||||
ctx := context.Background()
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -305,10 +305,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetryNoAuth returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
|
||||
// For 429 or 503 errors look at the Retry-After: header and
|
||||
// set the retry appropriately, starting with a minimum of 1
|
||||
// second if it isn't set.
|
||||
@@ -339,7 +336,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
}
|
||||
return true, err
|
||||
}
|
||||
return f.shouldRetryNoReauth(ctx, resp, err)
|
||||
return f.shouldRetryNoReauth(resp, err)
|
||||
}
|
||||
|
||||
// errorHandler parses a non 2xx error response into an error
|
||||
@@ -507,7 +504,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
|
||||
return f.shouldRetryNoReauth(ctx, resp, err)
|
||||
return f.shouldRetryNoReauth(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to authenticate")
|
||||
@@ -1744,13 +1741,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
|
||||
ContentType: resp.Header.Get("Content-Type"),
|
||||
Info: Info,
|
||||
}
|
||||
// When reading files from B2 via cloudflare using
|
||||
// --b2-download-url cloudflare strips the Content-Length
|
||||
// headers (presumably so it can inject stuff) so use the old
|
||||
// length read from the listing.
|
||||
if info.Size < 0 {
|
||||
info.Size = o.size
|
||||
}
|
||||
return resp, info, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -317,13 +317,10 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
authRetry := false
|
||||
|
||||
if resp != nil && resp.StatusCode == 401 && strings.Contains(resp.Header.Get("Www-Authenticate"), "expired_token") {
|
||||
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
|
||||
authRetry = true
|
||||
fs.Debugf(nil, "Should retry: %v", err)
|
||||
}
|
||||
@@ -551,7 +548,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -588,7 +585,7 @@ OUTER:
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -743,7 +740,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -770,7 +767,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rmdir failed")
|
||||
@@ -842,7 +839,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var info *api.Item
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, ©File, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -880,7 +877,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -898,7 +895,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read user info")
|
||||
@@ -1011,7 +1008,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return info.SharedLink.URL, err
|
||||
}
|
||||
@@ -1029,7 +1026,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1051,7 +1048,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't list trash")
|
||||
@@ -1185,7 +1182,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
|
||||
var info *api.Item
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return info, err
|
||||
}
|
||||
@@ -1218,7 +1215,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1258,7 +1255,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -44,7 +44,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -109,10 +109,10 @@ outer:
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
body, err = rest.ReadBody(resp)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
delay := defaultDelay
|
||||
var why string
|
||||
@@ -167,7 +167,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
6
backend/cache/cache_internal_test.go
vendored
6
backend/cache/cache_internal_test.go
vendored
@@ -892,7 +892,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
m.Set("type", "cache")
|
||||
m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
|
||||
} else {
|
||||
remoteType := config.FileGet(remote, "type")
|
||||
remoteType := config.FileGet(remote, "type", "")
|
||||
if remoteType == "" {
|
||||
t.Skipf("skipped due to invalid remote type for %v", remote)
|
||||
return nil, nil
|
||||
@@ -903,14 +903,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
m.Set("password", cryptPassword1)
|
||||
m.Set("password2", cryptPassword2)
|
||||
}
|
||||
remoteRemote := config.FileGet(remote, "remote")
|
||||
remoteRemote := config.FileGet(remote, "remote", "")
|
||||
if remoteRemote == "" {
|
||||
t.Skipf("skipped due to invalid remote wrapper for %v", remote)
|
||||
return nil, nil
|
||||
}
|
||||
remoteRemoteParts := strings.Split(remoteRemote, ":")
|
||||
remoteWrapping := remoteRemoteParts[0]
|
||||
remoteType := config.FileGet(remoteWrapping, "type")
|
||||
remoteType := config.FileGet(remoteWrapping, "type", "")
|
||||
if remoteType != "cache" {
|
||||
t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
|
||||
return nil, nil
|
||||
|
||||
@@ -277,10 +277,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
|
||||
return nil, errors.New("can't point remote at itself - check the value of the remote setting")
|
||||
}
|
||||
|
||||
baseName, basePath, err := fspath.SplitFs(remote)
|
||||
baseName, basePath, err := fspath.Parse(remote)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
|
||||
}
|
||||
if baseName != "" {
|
||||
baseName += ":"
|
||||
}
|
||||
// Look for a file first
|
||||
remotePath := fspath.JoinRootPath(basePath, rpath)
|
||||
baseFs, err := cache.Get(ctx, baseName+remotePath)
|
||||
|
||||
@@ -404,16 +404,13 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read destination hash")
|
||||
}
|
||||
if srcHash != "" && dstHash != "" {
|
||||
if srcHash != dstHash {
|
||||
// remove object
|
||||
err = o.Remove(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Failed to remove corrupted object: %v", err)
|
||||
}
|
||||
return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
|
||||
if srcHash != "" && dstHash != "" && srcHash != dstHash {
|
||||
// remove object
|
||||
err = o.Remove(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Failed to remove corrupted object: %v", err)
|
||||
}
|
||||
fs.Debugf(src, "%v = %s OK", ht, srcHash)
|
||||
return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -590,13 +590,13 @@ type Fs struct {
|
||||
}
|
||||
|
||||
type baseObject struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
id string // Drive Id of this object
|
||||
modifiedDate string // RFC3339 time it was last modified
|
||||
mimeType string // The object MIME type
|
||||
bytes int64 // size of the object
|
||||
parents []string // IDs of the parent directories
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
id string // Drive Id of this object
|
||||
modifiedDate string // RFC3339 time it was last modified
|
||||
mimeType string // The object MIME type
|
||||
bytes int64 // size of the object
|
||||
parents int // number of parents
|
||||
}
|
||||
type documentObject struct {
|
||||
baseObject
|
||||
@@ -641,10 +641,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
}
|
||||
|
||||
// shouldRetry determines whether a given err rates being retried
|
||||
func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(err error) (bool, error) {
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
@@ -698,20 +695,20 @@ func containsString(slice []string, s string) bool {
|
||||
}
|
||||
|
||||
// getFile returns drive.File for the ID passed and fields passed in
|
||||
func (f *Fs) getFile(ctx context.Context, ID string, fields googleapi.Field) (info *drive.File, err error) {
|
||||
func (f *Fs) getFile(ID string, fields googleapi.Field) (info *drive.File, err error) {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info, err = f.svc.Files.Get(ID).
|
||||
Fields(fields).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
return info, err
|
||||
}
|
||||
|
||||
// getRootID returns the canonical ID for the "root" ID
|
||||
func (f *Fs) getRootID(ctx context.Context) (string, error) {
|
||||
info, err := f.getFile(ctx, "root", "id")
|
||||
func (f *Fs) getRootID() (string, error) {
|
||||
info, err := f.getFile("root", "id")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "couldn't find root directory ID")
|
||||
}
|
||||
@@ -817,7 +814,7 @@ OUTER:
|
||||
var files *drive.FileList
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "couldn't list directory")
|
||||
@@ -840,7 +837,7 @@ OUTER:
|
||||
if filesOnly && item.ShortcutDetails.TargetMimeType == driveFolderType {
|
||||
continue
|
||||
}
|
||||
item, err = f.resolveShortcut(ctx, item)
|
||||
item, err = f.resolveShortcut(item)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "list")
|
||||
}
|
||||
@@ -858,7 +855,7 @@ OUTER:
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
_, exportName, _, _ := f.findExportFormat(ctx, item)
|
||||
_, exportName, _, _ := f.findExportFormat(item)
|
||||
if exportName == "" || exportName != title {
|
||||
continue
|
||||
}
|
||||
@@ -1158,7 +1155,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
|
||||
f.rootFolderID = f.opt.TeamDriveID
|
||||
} else {
|
||||
// otherwise look up the actual root ID
|
||||
rootID, err := f.getRootID(ctx)
|
||||
rootID, err := f.getRootID()
|
||||
if err != nil {
|
||||
if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
|
||||
// 404 means that this scope does not have permission to get the
|
||||
@@ -1239,7 +1236,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
|
||||
modifiedDate: modifiedDate,
|
||||
mimeType: info.MimeType,
|
||||
bytes: size,
|
||||
parents: info.Parents,
|
||||
parents: len(info.Parents),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1331,26 +1328,26 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
|
||||
// newObjectWithInfo creates an fs.Object for any drive.File
|
||||
//
|
||||
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
|
||||
// If item has MD5 sum or a length it is a file stored on drive
|
||||
if info.Md5Checksum != "" || info.Size > 0 {
|
||||
return f.newRegularObject(remote, info), nil
|
||||
}
|
||||
|
||||
extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
|
||||
return f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
|
||||
extension, exportName, exportMimeType, isDocument := f.findExportFormat(info)
|
||||
return f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
|
||||
}
|
||||
|
||||
// newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
|
||||
//
|
||||
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
||||
func (f *Fs) newObjectWithExportInfo(
|
||||
ctx context.Context, remote string, info *drive.File,
|
||||
remote string, info *drive.File,
|
||||
extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
|
||||
// Note that resolveShortcut will have been called already if
|
||||
// we are being called from a listing. However the drive.Item
|
||||
// will have been resolved so this will do nothing.
|
||||
info, err = f.resolveShortcut(ctx, info)
|
||||
info, err = f.resolveShortcut(info)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "new object")
|
||||
}
|
||||
@@ -1398,7 +1395,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
remote = remote[:len(remote)-len(extension)]
|
||||
obj, err := f.newObjectWithExportInfo(ctx, remote, info, extension, exportName, exportMimeType, isDocument)
|
||||
obj, err := f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, err
|
||||
@@ -1415,7 +1412,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
pathID = actualID(pathID)
|
||||
found, err = f.list(ctx, []string{pathID}, leaf, true, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
||||
if !f.opt.SkipGdocs {
|
||||
_, exportName, _, isDocument := f.findExportFormat(ctx, item)
|
||||
_, exportName, _, isDocument := f.findExportFormat(item)
|
||||
if exportName == leaf {
|
||||
pathIDOut = item.Id
|
||||
return true
|
||||
@@ -1450,8 +1447,8 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
info, err = f.svc.Files.Create(createInfo).
|
||||
Fields("id").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1486,15 +1483,15 @@ func linkTemplate(mt string) *template.Template {
|
||||
})
|
||||
return _linkTemplates[mt]
|
||||
}
|
||||
func (f *Fs) fetchFormats(ctx context.Context) {
|
||||
func (f *Fs) fetchFormats() {
|
||||
fetchFormatsOnce.Do(func() {
|
||||
var about *drive.About
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
about, err = f.svc.About.Get().
|
||||
Fields("exportFormats,importFormats").
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Failed to get Drive exportFormats and importFormats: %v", err)
|
||||
@@ -1511,8 +1508,8 @@ func (f *Fs) fetchFormats(ctx context.Context) {
|
||||
// if necessary.
|
||||
//
|
||||
// if the fetch fails then it will not export any drive formats
|
||||
func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
|
||||
f.fetchFormats(ctx)
|
||||
func (f *Fs) exportFormats() map[string][]string {
|
||||
f.fetchFormats()
|
||||
return _exportFormats
|
||||
}
|
||||
|
||||
@@ -1520,8 +1517,8 @@ func (f *Fs) exportFormats(ctx context.Context) map[string][]string {
|
||||
// if necessary.
|
||||
//
|
||||
// if the fetch fails then it will not import any drive formats
|
||||
func (f *Fs) importFormats(ctx context.Context) map[string][]string {
|
||||
f.fetchFormats(ctx)
|
||||
func (f *Fs) importFormats() map[string][]string {
|
||||
f.fetchFormats()
|
||||
return _importFormats
|
||||
}
|
||||
|
||||
@@ -1530,9 +1527,9 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string {
|
||||
//
|
||||
// Look through the exportExtensions and find the first format that can be
|
||||
// converted. If none found then return ("", "", false)
|
||||
func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) (
|
||||
func (f *Fs) findExportFormatByMimeType(itemMimeType string) (
|
||||
extension, mimeType string, isDocument bool) {
|
||||
exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType]
|
||||
exportMimeTypes, isDocument := f.exportFormats()[itemMimeType]
|
||||
if isDocument {
|
||||
for _, _extension := range f.exportExtensions {
|
||||
_mimeType := mime.TypeByExtension(_extension)
|
||||
@@ -1559,8 +1556,8 @@ func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string
|
||||
//
|
||||
// Look through the exportExtensions and find the first format that can be
|
||||
// converted. If none found then return ("", "", "", false)
|
||||
func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension, filename, mimeType string, isDocument bool) {
|
||||
extension, mimeType, isDocument = f.findExportFormatByMimeType(ctx, item.MimeType)
|
||||
func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
|
||||
extension, mimeType, isDocument = f.findExportFormatByMimeType(item.MimeType)
|
||||
if extension != "" {
|
||||
filename = item.Name + extension
|
||||
}
|
||||
@@ -1572,9 +1569,9 @@ func (f *Fs) findExportFormat(ctx context.Context, item *drive.File) (extension,
|
||||
// MIME type is returned
|
||||
//
|
||||
// When no match is found "" is returned.
|
||||
func (f *Fs) findImportFormat(ctx context.Context, mimeType string) string {
|
||||
func (f *Fs) findImportFormat(mimeType string) string {
|
||||
mimeType = fixMimeType(mimeType)
|
||||
ifs := f.importFormats(ctx)
|
||||
ifs := f.importFormats()
|
||||
for _, mt := range f.importMimeTypes {
|
||||
if mt == mimeType {
|
||||
importMimeTypes := ifs[mimeType]
|
||||
@@ -1607,7 +1604,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
|
||||
var iErr error
|
||||
_, err = f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
||||
entry, err := f.itemToDirEntry(ctx, path.Join(dir, item.Name), item)
|
||||
entry, err := f.itemToDirEntry(path.Join(dir, item.Name), item)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
@@ -1720,7 +1717,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
|
||||
}
|
||||
}
|
||||
remote := path.Join(paths[i], item.Name)
|
||||
entry, err := f.itemToDirEntry(ctx, remote, item)
|
||||
entry, err := f.itemToDirEntry(remote, item)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
@@ -1985,7 +1982,7 @@ func isShortcut(item *drive.File) bool {
|
||||
// Note that we assume shortcuts can't point to shortcuts. Google
|
||||
// drive web interface doesn't offer the option to create a shortcut
|
||||
// to a shortcut. The documentation is silent on the issue.
|
||||
func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *drive.File, err error) {
|
||||
func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error) {
|
||||
if f.opt.SkipShortcuts || item.MimeType != shortcutMimeType {
|
||||
return item, nil
|
||||
}
|
||||
@@ -1993,7 +1990,7 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
|
||||
fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
|
||||
return item, nil
|
||||
}
|
||||
newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
|
||||
newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
|
||||
if err != nil {
|
||||
if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
|
||||
// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
|
||||
@@ -2015,21 +2012,18 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
|
||||
// itemToDirEntry converts a drive.File to an fs.DirEntry.
|
||||
// When the drive.File cannot be represented as an fs.DirEntry
|
||||
// (nil, nil) is returned.
|
||||
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
||||
func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
||||
switch {
|
||||
case item.MimeType == driveFolderType:
|
||||
// cache the directory ID for later lookups
|
||||
f.dirCache.Put(remote, item.Id)
|
||||
when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
|
||||
d := fs.NewDir(remote, when).SetID(item.Id)
|
||||
if len(item.Parents) > 0 {
|
||||
d.SetParentID(item.Parents[0])
|
||||
}
|
||||
return d, nil
|
||||
case f.opt.AuthOwnerOnly && !isAuthOwned(item):
|
||||
// ignore object
|
||||
default:
|
||||
entry, err = f.newObjectWithInfo(ctx, remote, item)
|
||||
entry, err = f.newObjectWithInfo(remote, item)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -2096,12 +2090,12 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
importMimeType := ""
|
||||
|
||||
if f.importMimeTypes != nil && !f.opt.SkipGdocs {
|
||||
importMimeType = f.findImportFormat(ctx, srcMimeType)
|
||||
importMimeType = f.findImportFormat(srcMimeType)
|
||||
|
||||
if isInternalMimeType(importMimeType) {
|
||||
remote = remote[:len(remote)-len(srcExt)]
|
||||
|
||||
exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType)
|
||||
exportExt, _, _ = f.findExportFormatByMimeType(importMimeType)
|
||||
if exportExt == "" {
|
||||
return nil, errors.Errorf("No export format found for %q", importMimeType)
|
||||
}
|
||||
@@ -2131,8 +2125,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2144,7 +2138,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return f.newObjectWithInfo(ctx, remote, info)
|
||||
return f.newObjectWithInfo(remote, info)
|
||||
}
|
||||
|
||||
// MergeDirs merges the contents of all the directories passed
|
||||
@@ -2186,8 +2180,8 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
AddParents(dstDir.ID()).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
|
||||
@@ -2220,14 +2214,14 @@ func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
|
||||
_, err = f.svc.Files.Update(id, &info).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
Do()
|
||||
} else {
|
||||
err = f.svc.Files.Delete(id).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
Do()
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2340,12 +2334,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
if isDoc {
|
||||
// preserve the description on copy for docs
|
||||
info, err := f.getFile(ctx, actualID(srcObj.id), "description")
|
||||
info, err := f.getFile(actualID(srcObj.id), "description")
|
||||
if err != nil {
|
||||
fs.Errorf(srcObj, "Failed to read description for Google Doc: %v", err)
|
||||
} else {
|
||||
createInfo.Description = info.Description
|
||||
return nil, errors.Wrap(err, "failed to read description for Google Doc")
|
||||
}
|
||||
createInfo.Description = info.Description
|
||||
} else {
|
||||
// don't overwrite the description on copy for files
|
||||
// this should work for docs but it doesn't - it is probably a bug in Google Drive
|
||||
@@ -2361,13 +2354,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newObject, err := f.newObjectWithInfo(ctx, remote, info)
|
||||
newObject, err := f.newObjectWithInfo(remote, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2461,7 +2454,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
err := f.svc.Files.EmptyTrash().Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -2479,7 +2472,7 @@ func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
|
||||
var td *drive.Drive
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
td, err = f.svc.Drives.Get(f.opt.TeamDriveID).Fields("name,id,capabilities,createdTime,restrictions").Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get Shared Drive info")
|
||||
@@ -2502,7 +2495,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
about, err = f.svc.About.Get().Fields("storageQuota").Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get Drive storageQuota")
|
||||
@@ -2574,14 +2567,14 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
AddParents(dstParents).
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.newObjectWithInfo(ctx, remote, info)
|
||||
return f.newObjectWithInfo(remote, info)
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
@@ -2611,8 +2604,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
_, err = f.svc.Permissions.Create(id, permission).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -2654,8 +2647,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
AddParents(dstDirectoryID).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -2673,7 +2666,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
||||
go func() {
|
||||
// get the StartPageToken early so all changes from now on get processed
|
||||
startPageToken, err := f.changeNotifyStartPageToken(ctx)
|
||||
startPageToken, err := f.changeNotifyStartPageToken()
|
||||
if err != nil {
|
||||
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
||||
}
|
||||
@@ -2698,7 +2691,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
case <-tickerC:
|
||||
if startPageToken == "" {
|
||||
startPageToken, err = f.changeNotifyStartPageToken(ctx)
|
||||
startPageToken, err = f.changeNotifyStartPageToken()
|
||||
if err != nil {
|
||||
fs.Infof(f, "Failed to get StartPageToken: %s", err)
|
||||
continue
|
||||
@@ -2713,15 +2706,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
}()
|
||||
}
|
||||
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) {
|
||||
func (f *Fs) changeNotifyStartPageToken() (pageToken string, err error) {
|
||||
var startPageToken *drive.StartPageToken
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changes := f.svc.Changes.GetStartPageToken().SupportsAllDrives(true)
|
||||
if f.isTeamDrive {
|
||||
changes.DriveId(f.opt.TeamDriveID)
|
||||
}
|
||||
startPageToken, err = changes.Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
startPageToken, err = changes.Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -2750,7 +2743,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
changesCall.Spaces("appDataFolder")
|
||||
}
|
||||
changeList, err = changesCall.Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -2946,8 +2939,8 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(dstFs.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
return dstFs.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return dstFs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "shortcut creation failed")
|
||||
@@ -2955,7 +2948,7 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
|
||||
if isDir {
|
||||
return nil, nil
|
||||
}
|
||||
return dstFs.newObjectWithInfo(ctx, dstPath, info)
|
||||
return dstFs.newObjectWithInfo(dstPath, info)
|
||||
}
|
||||
|
||||
// List all team drives
|
||||
@@ -2967,7 +2960,7 @@ func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err
|
||||
var teamDrives *drive.TeamDriveList
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
teamDrives, err = listTeamDrives.Context(ctx).Do()
|
||||
return defaultFs.shouldRetry(ctx, err)
|
||||
return defaultFs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return drives, errors.Wrap(err, "listing Team Drives failed")
|
||||
@@ -3009,8 +3002,8 @@ func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurs
|
||||
_, err := f.svc.Files.Update(item.Id, &update).
|
||||
SupportsAllDrives(true).
|
||||
Fields("trashed").
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "failed to restore")
|
||||
@@ -3052,7 +3045,7 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
|
||||
|
||||
// copy file with id to dest
|
||||
func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
|
||||
info, err := f.getFile(ctx, id, f.fileFields)
|
||||
info, err := f.getFile(id, f.fileFields)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't find id")
|
||||
}
|
||||
@@ -3060,7 +3053,7 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
|
||||
return errors.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
|
||||
}
|
||||
info.Name = f.opt.Enc.ToStandardName(info.Name)
|
||||
o, err := f.newObjectWithInfo(ctx, info.Name, info)
|
||||
o, err := f.newObjectWithInfo(info.Name, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3361,7 +3354,7 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
|
||||
|
||||
found, err := f.list(ctx, []string{directoryID}, leaf, false, false, f.opt.TrashedOnly, false, func(item *drive.File) bool {
|
||||
if !f.opt.SkipGdocs {
|
||||
extension, exportName, exportMimeType, isDocument = f.findExportFormat(ctx, item)
|
||||
extension, exportName, exportMimeType, isDocument = f.findExportFormat(item)
|
||||
if exportName == leaf {
|
||||
info = item
|
||||
return true
|
||||
@@ -3412,8 +3405,8 @@ func (o *baseObject) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo).
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -3451,7 +3444,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
|
||||
_ = res.Body.Close() // ignore error
|
||||
}
|
||||
}
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return req, nil, err
|
||||
@@ -3543,8 +3536,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
v2File, err = o.fs.v2Svc.Files.Get(actualID(o.id)).
|
||||
Fields("downloadUrl").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
|
||||
@@ -3624,8 +3617,8 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(o.fs.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
Do()
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -3668,7 +3661,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newO, err := o.fs.newObjectWithInfo(ctx, src.Remote(), info)
|
||||
newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3692,7 +3685,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
|
||||
if o.fs.importMimeTypes == nil || o.fs.opt.SkipGdocs {
|
||||
return errors.Errorf("can't update google document type without --drive-import-formats")
|
||||
}
|
||||
importMimeType = o.fs.findImportFormat(ctx, updateInfo.MimeType)
|
||||
importMimeType = o.fs.findImportFormat(updateInfo.MimeType)
|
||||
if importMimeType == "" {
|
||||
return errors.Errorf("no import format found for %q", srcMimeType)
|
||||
}
|
||||
@@ -3709,7 +3702,7 @@ func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.Object
|
||||
remote := src.Remote()
|
||||
remote = remote[:len(remote)-o.extLen]
|
||||
|
||||
newO, err := o.fs.newObjectWithInfo(ctx, remote, info)
|
||||
newO, err := o.fs.newObjectWithInfo(remote, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3729,7 +3722,7 @@ func (o *linkObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo
|
||||
|
||||
// Remove an object
|
||||
func (o *baseObject) Remove(ctx context.Context) error {
|
||||
if len(o.parents) > 1 {
|
||||
if o.parents > 1 {
|
||||
return errors.New("can't delete safely - has multiple parents")
|
||||
}
|
||||
return o.fs.delete(ctx, shortcutID(o.id), o.fs.opt.UseTrash)
|
||||
@@ -3745,14 +3738,6 @@ func (o *baseObject) ID() string {
|
||||
return o.id
|
||||
}
|
||||
|
||||
// ParentID returns the ID of the Object parent if known, or "" if not
|
||||
func (o *baseObject) ParentID() string {
|
||||
if len(o.parents) > 0 {
|
||||
return o.parents[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (o *documentObject) ext() string {
|
||||
return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
|
||||
}
|
||||
@@ -3813,13 +3798,10 @@ var (
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
_ fs.ParentIDer = (*Object)(nil)
|
||||
_ fs.Object = (*documentObject)(nil)
|
||||
_ fs.MimeTyper = (*documentObject)(nil)
|
||||
_ fs.IDer = (*documentObject)(nil)
|
||||
_ fs.ParentIDer = (*documentObject)(nil)
|
||||
_ fs.Object = (*linkObject)(nil)
|
||||
_ fs.MimeTyper = (*linkObject)(nil)
|
||||
_ fs.IDer = (*linkObject)(nil)
|
||||
_ fs.ParentIDer = (*linkObject)(nil)
|
||||
)
|
||||
|
||||
@@ -111,7 +111,6 @@ func TestInternalParseExtensions(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInternalFindExportFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
item := &drive.File{
|
||||
Name: "file",
|
||||
MimeType: "application/vnd.google-apps.document",
|
||||
@@ -129,7 +128,7 @@ func TestInternalFindExportFormat(t *testing.T) {
|
||||
} {
|
||||
f := new(Fs)
|
||||
f.exportExtensions = test.extensions
|
||||
gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
|
||||
gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
|
||||
assert.Equal(t, test.wantExtension, gotExtension)
|
||||
if test.wantExtension != "" {
|
||||
assert.Equal(t, item.Name+gotExtension, gotFilename)
|
||||
|
||||
@@ -94,7 +94,7 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
defer googleapi.CloseBody(res)
|
||||
err = googleapi.CheckResponse(res)
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -202,7 +202,7 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
again, err := rx.f.shouldRetry(err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
err = nil
|
||||
|
||||
@@ -292,10 +292,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(err error) (bool, error) {
|
||||
if err == nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -310,7 +307,7 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
switch e := err.(type) {
|
||||
case auth.RateLimitAPIError:
|
||||
if e.RateLimitError.RetryAfter > 0 {
|
||||
fs.Logf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
|
||||
fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
|
||||
err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
|
||||
}
|
||||
return true, err
|
||||
@@ -428,7 +425,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if f.root == "" {
|
||||
return f, nil
|
||||
}
|
||||
_, err := f.findSharedFile(ctx, f.root)
|
||||
_, err := f.findSharedFile(f.root)
|
||||
f.root = ""
|
||||
if err == nil {
|
||||
return f, fs.ErrorIsFile
|
||||
@@ -448,7 +445,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// root is not empty so we have find the right shared folder if it exists
|
||||
id, err := f.findSharedFolder(ctx, dir)
|
||||
id, err := f.findSharedFolder(dir)
|
||||
if err != nil {
|
||||
// if we didn't find the specified shared folder we have to bail out here
|
||||
return nil, err
|
||||
@@ -456,7 +453,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// we found the specified shared folder so let's mount it
|
||||
// this will add it to the users normal root namespace and allows us
|
||||
// to actually perform operations on it using the normal api endpoints.
|
||||
err = f.mountSharedFolder(ctx, id)
|
||||
err = f.mountSharedFolder(id)
|
||||
if err != nil {
|
||||
switch e := err.(type) {
|
||||
case sharing.MountFolderAPIError:
|
||||
@@ -480,7 +477,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
var acc *users.FullAccount
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
acc, err = f.users.GetCurrentAccount()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "get current account failed")
|
||||
@@ -498,7 +495,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.setRoot(root)
|
||||
|
||||
// See if the root is actually an object
|
||||
_, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
_, err = f.getFileMetadata(f.slashRoot)
|
||||
if err == nil {
|
||||
newRoot := path.Dir(f.root)
|
||||
if newRoot == "." {
|
||||
@@ -532,12 +529,12 @@ func (f *Fs) setRoot(root string) {
|
||||
}
|
||||
|
||||
// getMetadata gets the metadata for a file or directory
|
||||
func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
|
||||
func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
|
||||
Path: f.opt.Enc.FromStandardPath(objPath),
|
||||
})
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
switch e := err.(type) {
|
||||
@@ -552,8 +549,8 @@ func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMet
|
||||
}
|
||||
|
||||
// getFileMetadata gets the metadata for a file
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
|
||||
entry, notFound, err := f.getMetadata(ctx, filePath)
|
||||
func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
|
||||
entry, notFound, err := f.getMetadata(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -568,8 +565,8 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *fi
|
||||
}
|
||||
|
||||
// getDirMetadata gets the metadata for a directory
|
||||
func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
|
||||
entry, notFound, err := f.getMetadata(ctx, dirPath)
|
||||
func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
|
||||
entry, notFound, err := f.getMetadata(dirPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -586,7 +583,7 @@ func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) {
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
@@ -595,7 +592,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.F
|
||||
if info != nil {
|
||||
err = o.setMetadataFromEntry(info)
|
||||
} else {
|
||||
err = o.readEntryAndSetMetadata(ctx)
|
||||
err = o.readEntryAndSetMetadata()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -607,14 +604,14 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.F
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if f.opt.SharedFiles {
|
||||
return f.findSharedFile(ctx, remote)
|
||||
return f.findSharedFile(remote)
|
||||
}
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
return f.newObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// listSharedFoldersApi lists all available shared folders mounted and not mounted
|
||||
// we'll need the id later so we have to return them in original format
|
||||
func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
|
||||
func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
|
||||
started := false
|
||||
var res *sharing.ListFoldersResult
|
||||
for {
|
||||
@@ -624,7 +621,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFolders(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -636,7 +633,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "list continue")
|
||||
@@ -661,8 +658,8 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err
|
||||
// findSharedFolder find the id for a given shared folder name
|
||||
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
|
||||
// so our only option is to iterate over all shared folders
|
||||
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
|
||||
entries, err := f.listSharedFolders(ctx)
|
||||
func (f *Fs) findSharedFolder(name string) (id string, err error) {
|
||||
entries, err := f.listSharedFolders()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -675,20 +672,20 @@ func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err
|
||||
}
|
||||
|
||||
// mountSharedFolder mount a shared folder to the root namespace
|
||||
func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
|
||||
func (f *Fs) mountSharedFolder(id string) error {
|
||||
arg := sharing.MountFolderArg{
|
||||
SharedFolderId: id,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.sharing.MountFolder(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// listReceivedFiles lists shared the user as access to (note this means individual
|
||||
// files not files contained in shared folders)
|
||||
func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
|
||||
func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
|
||||
started := false
|
||||
var res *sharing.ListFilesResult
|
||||
for {
|
||||
@@ -698,7 +695,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFiles(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -710,7 +707,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "list continue")
|
||||
@@ -737,8 +734,8 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
|
||||
files, err := f.listReceivedFiles(ctx)
|
||||
func (f *Fs) findSharedFile(name string) (o *Object, err error) {
|
||||
files, err := f.listReceivedFiles()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -761,10 +758,10 @@ func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err er
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
if f.opt.SharedFiles {
|
||||
return f.listReceivedFiles(ctx)
|
||||
return f.listReceivedFiles()
|
||||
}
|
||||
if f.opt.SharedFolders {
|
||||
return f.listSharedFolders(ctx)
|
||||
return f.listSharedFolders()
|
||||
}
|
||||
|
||||
root := f.slashRoot
|
||||
@@ -785,7 +782,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolder(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
switch e := err.(type) {
|
||||
@@ -803,7 +800,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "list continue")
|
||||
@@ -833,7 +830,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
|
||||
entries = append(entries, d)
|
||||
} else if fileInfo != nil {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
|
||||
o, err := f.newObjectWithInfo(remote, fileInfo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -882,7 +879,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
}
|
||||
|
||||
// check directory doesn't exist
|
||||
_, err := f.getDirMetadata(ctx, root)
|
||||
_, err := f.getDirMetadata(root)
|
||||
if err == nil {
|
||||
return nil // directory exists already
|
||||
} else if err != fs.ErrorDirNotFound {
|
||||
@@ -899,7 +896,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.CreateFolderV2(&arg2)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -916,7 +913,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
|
||||
if check {
|
||||
// check directory exists
|
||||
_, err = f.getDirMetadata(ctx, root)
|
||||
_, err = f.getDirMetadata(root)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Rmdir")
|
||||
}
|
||||
@@ -933,7 +930,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
var res *files.ListFolderResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolder(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Rmdir")
|
||||
@@ -946,7 +943,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
// remove it
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -999,7 +996,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.CopyV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "copy failed")
|
||||
@@ -1060,7 +1057,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.MoveV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "move failed")
|
||||
@@ -1094,7 +1091,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var linkRes sharing.IsSharedLinkMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil && strings.Contains(err.Error(),
|
||||
@@ -1107,7 +1104,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var listRes *sharing.ListSharedLinksResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
listRes, err = f.sharing.ListSharedLinks(&listArg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -1149,7 +1146,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
dstPath := path.Join(f.slashRoot, dstRemote)
|
||||
|
||||
// Check if destination exists
|
||||
_, err := f.getDirMetadata(ctx, dstPath)
|
||||
_, err := f.getDirMetadata(dstPath)
|
||||
if err == nil {
|
||||
return fs.ErrorDirExists
|
||||
} else if err != fs.ErrorDirNotFound {
|
||||
@@ -1168,7 +1165,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.MoveV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "MoveDir failed")
|
||||
@@ -1182,7 +1179,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var q *users.SpaceUsage
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
q, err = f.users.GetSpaceUsage()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "about failed")
|
||||
@@ -1213,7 +1210,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
||||
go func() {
|
||||
// get the StartCursor early so all changes from now on get processed
|
||||
startCursor, err := f.changeNotifyCursor(ctx)
|
||||
startCursor, err := f.changeNotifyCursor()
|
||||
if err != nil {
|
||||
fs.Infof(f, "Failed to get StartCursor: %s", err)
|
||||
}
|
||||
@@ -1238,7 +1235,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
case <-tickerC:
|
||||
if startCursor == "" {
|
||||
startCursor, err = f.changeNotifyCursor(ctx)
|
||||
startCursor, err = f.changeNotifyCursor()
|
||||
if err != nil {
|
||||
fs.Infof(f, "Failed to get StartCursor: %s", err)
|
||||
continue
|
||||
@@ -1254,7 +1251,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}()
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) {
|
||||
func (f *Fs) changeNotifyCursor() (cursor string, err error) {
|
||||
var startCursor *files.ListFolderGetLatestCursorResult
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -1269,7 +1266,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
|
||||
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -1299,7 +1296,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
}
|
||||
|
||||
res, err = f.svc.ListFolderLongpoll(&args)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -1322,7 +1319,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changeList, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "list continue")
|
||||
@@ -1395,7 +1392,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != DbHashType {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
err := o.readMetaData(ctx)
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to read hash from metadata")
|
||||
}
|
||||
@@ -1419,17 +1416,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
|
||||
}
|
||||
|
||||
// Reads the entry for a file from dropbox
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
|
||||
return o.fs.getFileMetadata(ctx, o.remotePath())
|
||||
func (o *Object) readEntry() (*files.FileMetadata, error) {
|
||||
return o.fs.getFileMetadata(o.remotePath())
|
||||
}
|
||||
|
||||
// Read entry if not set and set metadata from it
|
||||
func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
|
||||
func (o *Object) readEntryAndSetMetadata() error {
|
||||
// Last resort set time from client
|
||||
if !o.modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
entry, err := o.readEntry(ctx)
|
||||
entry, err := o.readEntry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1442,12 +1439,12 @@ func (o *Object) remotePath() string {
|
||||
}
|
||||
|
||||
// readMetaData gets the info if it hasn't already been fetched
|
||||
func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
func (o *Object) readMetaData() (err error) {
|
||||
if !o.modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
// Last resort
|
||||
return o.readEntryAndSetMetadata(ctx)
|
||||
return o.readEntryAndSetMetadata()
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
@@ -1455,7 +1452,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
err := o.readMetaData(ctx)
|
||||
err := o.readMetaData()
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to read metadata: %v", err)
|
||||
return time.Now()
|
||||
@@ -1489,7 +1486,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1505,7 +1502,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.srv.Download(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
|
||||
switch e := err.(type) {
|
||||
@@ -1524,7 +1521,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// Will work optimally if size is >= uploadChunkSize. If the size is either
|
||||
// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
|
||||
// avoidable request to the Dropbox API that does not carry payload.
|
||||
func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
|
||||
func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
chunks := 0
|
||||
if size != -1 {
|
||||
@@ -1553,7 +1550,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
return false, nil
|
||||
}
|
||||
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1679,11 +1676,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var err error
|
||||
var entry *files.FileMetadata
|
||||
if size > int64(o.fs.opt.ChunkSize) || size == -1 {
|
||||
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
|
||||
entry, err = o.uploadChunked(in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
entry, err = o.fs.srv.Upload(commitInfo, in)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1701,7 +1698,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
|
||||
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
|
||||
})
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -28,10 +28,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// Detect this error which the integration tests provoke
|
||||
// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
|
||||
//
|
||||
@@ -77,7 +74,7 @@ func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
|
||||
var file File
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't read file info")
|
||||
@@ -99,7 +96,7 @@ func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenRespons
|
||||
var token GetTokenResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't list files")
|
||||
@@ -127,7 +124,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
|
||||
var sharedFiles SharedFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't list files")
|
||||
@@ -156,7 +153,7 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
|
||||
filesList = &FilesList{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't list files")
|
||||
@@ -184,7 +181,7 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
|
||||
foldersList = &FoldersList{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't list folders")
|
||||
@@ -278,7 +275,7 @@ func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (respons
|
||||
response = &MakeFolderResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't create folder")
|
||||
@@ -305,7 +302,7 @@ func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (respo
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.rest.CallJSON(ctx, &opts, request, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't remove folder")
|
||||
@@ -334,7 +331,7 @@ func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKRes
|
||||
response = &GenericOKResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -361,7 +358,7 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
|
||||
response = &MoveFileResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -386,7 +383,7 @@ func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename stri
|
||||
response = &CopyFileResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, request, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -408,7 +405,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
|
||||
response = &GetUploadNodeResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "didnt got an upload node")
|
||||
@@ -451,7 +448,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
|
||||
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -485,7 +482,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re
|
||||
response = &EndFileUploadResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -94,7 +94,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -228,10 +228,7 @@ var retryStatusCodes = []struct {
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, status api.OKError) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error, status api.OKError) (bool, error) {
|
||||
if err != nil {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
@@ -404,7 +401,7 @@ func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKEr
|
||||
// Refresh the body each retry
|
||||
opts.Body = strings.NewReader(data.Encode())
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, result)
|
||||
return f.shouldRetry(ctx, resp, err, result)
|
||||
return f.shouldRetry(resp, err, result)
|
||||
})
|
||||
if err != nil {
|
||||
return resp, err
|
||||
@@ -1280,7 +1277,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &uploader)
|
||||
return o.fs.shouldRetry(ctx, resp, err, nil)
|
||||
return o.fs.shouldRetry(resp, err, nil)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload")
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
@@ -34,12 +33,6 @@ var (
|
||||
currentUser = env.CurrentUser()
|
||||
)
|
||||
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
@@ -111,11 +104,6 @@ given, rclone will empty the connection pool.
|
||||
Set to 0 to keep connections indefinitely.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "close_timeout",
|
||||
Help: "Maximum time to wait for a response to close.",
|
||||
Default: fs.Duration(60 * time.Second),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -144,7 +132,6 @@ type Options struct {
|
||||
DisableEPSV bool `config:"disable_epsv"`
|
||||
DisableMLSD bool `config:"disable_mlsd"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
CloseTimeout fs.Duration `config:"close_timeout"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
@@ -164,7 +151,6 @@ type Fs struct {
|
||||
drain *time.Timer // used to drain the pool when we stop using the connections
|
||||
tokens *pacer.TokenDispenser
|
||||
tlsConf *tls.Config
|
||||
pacer *fs.Pacer // pacer for FTP connections
|
||||
}
|
||||
|
||||
// Object describes an FTP file
|
||||
@@ -258,24 +244,8 @@ func (d *dialCtx) dial(network, address string) (net.Conn, error) {
|
||||
return conn, err
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserve to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
switch errX := err.(type) {
|
||||
case *textproto.Error:
|
||||
switch errX.Code {
|
||||
case ftp.StatusNotAvailable:
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
// Open a new connection to the FTP server.
|
||||
func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
||||
func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
|
||||
fs.Debugf(f, "Connecting to FTP server")
|
||||
dCtx := dialCtx{f, ctx}
|
||||
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dCtx.dial)}
|
||||
@@ -297,22 +267,18 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
||||
if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
c, err = ftp.Dial(f.dialAddr, ftpConfig...)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
err = c.Login(f.user, f.pass)
|
||||
if err != nil {
|
||||
_ = c.Quit()
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
c, err := ftp.Dial(f.dialAddr, ftpConfig...)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "failed to make FTP connection to %q", f.dialAddr)
|
||||
fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
|
||||
return nil, errors.Wrap(err, "ftpConnection Dial")
|
||||
}
|
||||
return c, err
|
||||
err = c.Login(f.user, f.pass)
|
||||
if err != nil {
|
||||
_ = c.Quit()
|
||||
fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
|
||||
return nil, errors.Wrap(err, "ftpConnection Login")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Get an FTP connection from the pool, or open a new one
|
||||
@@ -445,7 +411,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
|
||||
dialAddr: dialAddr,
|
||||
tokens: pacer.NewTokenDispenser(opt.Concurrency),
|
||||
tlsConf: tlsConfig,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
@@ -966,8 +931,8 @@ func (f *ftpReadCloser) Close() error {
|
||||
go func() {
|
||||
errchan <- f.rc.Close()
|
||||
}()
|
||||
// Wait for Close for up to 60 seconds by default
|
||||
timer := time.NewTimer(time.Duration(f.f.opt.CloseTimeout))
|
||||
// Wait for Close for up to 60 seconds
|
||||
timer := time.NewTimer(60 * time.Second)
|
||||
select {
|
||||
case err = <-errchan:
|
||||
timer.Stop()
|
||||
|
||||
@@ -329,10 +329,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
}
|
||||
|
||||
// shouldRetry determines whether a given err rates being retried
|
||||
func shouldRetry(ctx context.Context, err error) (again bool, errOut error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(err error) (again bool, errOut error) {
|
||||
again = false
|
||||
if err != nil {
|
||||
if fserrors.ShouldRetry(err) {
|
||||
@@ -458,7 +455,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
newRoot := path.Dir(f.root)
|
||||
@@ -524,7 +521,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
var objects *storage.Objects
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
objects, err = list.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if gErr, ok := err.(*googleapi.Error); ok {
|
||||
@@ -627,7 +624,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
|
||||
var buckets *storage.Buckets
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
buckets, err = listBuckets.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -753,7 +750,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
|
||||
// service account that only has the "Storage Object Admin" role. See #2193 for details.
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
// Bucket already exists
|
||||
@@ -788,7 +785,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
|
||||
insertBucket.PredefinedAcl(f.opt.BucketACL)
|
||||
}
|
||||
_, err = insertBucket.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}, nil)
|
||||
}
|
||||
@@ -805,7 +802,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
|
||||
return f.cache.Remove(bucket, func() error {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -851,7 +848,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
for {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
rewriteResponse, err = rewriteRequest.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -944,7 +941,7 @@ func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, er
|
||||
bucket, bucketPath := o.split()
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if gErr, ok := err.(*googleapi.Error); ok {
|
||||
@@ -1015,7 +1012,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
|
||||
copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
|
||||
}
|
||||
newObject, err = copyObject.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1046,7 +1043,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
_ = res.Body.Close() // ignore error
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1112,7 +1109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
|
||||
}
|
||||
newObject, err = insertObject.Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1127,7 +1124,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
bucket, bucketPath := o.split()
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -240,10 +240,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -332,7 +329,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
|
||||
var openIDconfig map[string]interface{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "couldn't read openID config")
|
||||
@@ -361,7 +358,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't read user info")
|
||||
@@ -392,7 +389,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
|
||||
var res interface{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't revoke token")
|
||||
@@ -479,7 +476,7 @@ func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err erro
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't list albums")
|
||||
@@ -534,7 +531,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't list files")
|
||||
@@ -678,7 +675,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't create album")
|
||||
@@ -813,7 +810,7 @@ func (o *Object) Size() int64 {
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Reading size failed: %v", err)
|
||||
@@ -864,7 +861,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't get media item")
|
||||
@@ -941,7 +938,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -996,10 +993,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
token, err = rest.ReadBody(resp)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't upload file")
|
||||
@@ -1027,7 +1024,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var result api.BatchCreateResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create media item")
|
||||
@@ -1072,7 +1069,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't delete item from album")
|
||||
|
||||
@@ -109,7 +109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
dirname := path.Dir(realpath)
|
||||
fs.Debugf(o.fs, "update [%s]", realpath)
|
||||
|
||||
err := o.fs.client.MkdirAll(dirname, 0755)
|
||||
err := o.fs.client.MkdirAll(dirname, 755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
|
||||
ts := httptest.NewServer(handler)
|
||||
|
||||
// Configure the remote
|
||||
configfile.LoadConfig(context.Background())
|
||||
config.LoadConfig(context.Background())
|
||||
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||
// fs.Config.DumpHeaders = true
|
||||
// fs.Config.DumpBodies = true
|
||||
|
||||
@@ -235,10 +235,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -618,7 +615,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
@@ -857,7 +854,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -886,7 +883,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
var result api.JottaFolder
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -998,7 +995,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
@@ -1104,7 +1101,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't purge directory")
|
||||
@@ -1143,7 +1140,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1271,7 +1268,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var result api.JottaFile
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
@@ -1449,7 +1446,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1562,7 +1559,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var response api.AllocateFileResponse
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1627,7 +1624,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -42,9 +42,8 @@ func init() {
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Options: []fs.Option{{
|
||||
Name: "nounc",
|
||||
Help: "Disable UNC (long path names) conversion on Windows",
|
||||
Advanced: runtime.GOOS != "windows",
|
||||
Name: "nounc",
|
||||
Help: "Disable UNC (long path names) conversion on Windows",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "true",
|
||||
Help: "Disables long file names",
|
||||
@@ -1145,10 +1144,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err = file.PreAllocate(src.Size(), f)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to pre-allocate: %v", err)
|
||||
if err == file.ErrDiskFull {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
out = f
|
||||
|
||||
@@ -234,10 +234,7 @@ var retryErrorCodes = []int{
|
||||
// shouldRetry returns a boolean as to whether this response and err
|
||||
// deserve to be retried. It returns the err as a convenience.
|
||||
// Retries password authorization (once) in a special case of access denied.
|
||||
func shouldRetry(ctx context.Context, res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, error) {
|
||||
if res != nil && res.StatusCode == 403 && f.opt.Password != "" && !f.passFailed {
|
||||
reAuthErr := f.reAuthorize(opts, err)
|
||||
return reAuthErr == nil, err // return an original error
|
||||
@@ -603,7 +600,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
|
||||
var info api.ItemInfoResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -739,7 +736,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int)
|
||||
)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -803,7 +800,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
|
||||
var res *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
closeBody(res)
|
||||
@@ -1076,7 +1073,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
|
||||
var res *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
closeBody(res)
|
||||
@@ -1219,7 +1216,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
|
||||
var response api.GenericResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
|
||||
switch {
|
||||
@@ -1291,7 +1288,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var response api.GenericBodyResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -1395,7 +1392,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
|
||||
var res *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
closeBody(res)
|
||||
@@ -1486,7 +1483,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var response api.GenericBodyResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
|
||||
if err == nil && response.Body != "" {
|
||||
@@ -1527,7 +1524,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
var response api.CleanupResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1560,7 +1557,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var info api.UserInfoResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, res, err, f, &opts)
|
||||
return shouldRetry(res, err, f, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2079,7 +2076,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
|
||||
var res *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, res, err, o.fs, &opts)
|
||||
return shouldRetry(res, err, o.fs, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
closeBody(res)
|
||||
@@ -2175,7 +2172,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
opts.RootURL = server
|
||||
res, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, res, err, o.fs, &opts)
|
||||
return shouldRetry(res, err, o.fs, &opts)
|
||||
})
|
||||
if err != nil {
|
||||
if res != nil && res.Body != nil {
|
||||
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
@@ -159,10 +158,7 @@ func parsePath(path string) (root string) {
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(err error) (bool, error) {
|
||||
// Let the mega library handle the low level retries
|
||||
return false, err
|
||||
/*
|
||||
@@ -175,8 +171,8 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, remote string) (info *mega.Node, err error) {
|
||||
rootNode, err := f.findRoot(ctx, false)
|
||||
func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) {
|
||||
rootNode, err := f.findRoot(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -241,7 +237,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}).Fill(ctx, f)
|
||||
|
||||
// Find the root node and check if it is a file or not
|
||||
_, err = f.findRoot(ctx, false)
|
||||
_, err = f.findRoot(false)
|
||||
switch err {
|
||||
case nil:
|
||||
// root node found and is a directory
|
||||
@@ -311,8 +307,8 @@ func (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err
|
||||
// lookupDir looks up the node for the directory of the name given
|
||||
//
|
||||
// if create is true it tries to create the root directory if not found
|
||||
func (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {
|
||||
rootNode, err := f.findRoot(ctx, false)
|
||||
func (f *Fs) lookupDir(dir string) (*mega.Node, error) {
|
||||
rootNode, err := f.findRoot(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -320,15 +316,15 @@ func (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {
|
||||
}
|
||||
|
||||
// lookupParentDir finds the parent node for the remote passed in
|
||||
func (f *Fs) lookupParentDir(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
|
||||
func (f *Fs) lookupParentDir(remote string) (dirNode *mega.Node, leaf string, err error) {
|
||||
parent, leaf := path.Split(remote)
|
||||
dirNode, err = f.lookupDir(ctx, parent)
|
||||
dirNode, err = f.lookupDir(parent)
|
||||
return dirNode, leaf, err
|
||||
}
|
||||
|
||||
// mkdir makes the directory and any parent directories for the
|
||||
// directory of the name given
|
||||
func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *mega.Node, err error) {
|
||||
func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error) {
|
||||
f.mkdirMu.Lock()
|
||||
defer f.mkdirMu.Unlock()
|
||||
|
||||
@@ -362,7 +358,7 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *
|
||||
// create directory called name in node
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
node, err = f.srv.CreateDir(name, node)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "mkdir create node failed")
|
||||
@@ -372,20 +368,20 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *
|
||||
}
|
||||
|
||||
// mkdirParent creates the parent directory of remote
|
||||
func (f *Fs) mkdirParent(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {
|
||||
rootNode, err := f.findRoot(ctx, true)
|
||||
func (f *Fs) mkdirParent(remote string) (dirNode *mega.Node, leaf string, err error) {
|
||||
rootNode, err := f.findRoot(true)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
parent, leaf := path.Split(remote)
|
||||
dirNode, err = f.mkdir(ctx, rootNode, parent)
|
||||
dirNode, err = f.mkdir(rootNode, parent)
|
||||
return dirNode, leaf, err
|
||||
}
|
||||
|
||||
// findRoot looks up the root directory node and returns it.
|
||||
//
|
||||
// if create is true it tries to create the root directory if not found
|
||||
func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
|
||||
func (f *Fs) findRoot(create bool) (*mega.Node, error) {
|
||||
f.rootNodeMu.Lock()
|
||||
defer f.rootNodeMu.Unlock()
|
||||
|
||||
@@ -407,7 +403,7 @@ func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
|
||||
}
|
||||
|
||||
//..not found so create the root directory
|
||||
f._rootNode, err = f.mkdir(ctx, absRoot, f.root)
|
||||
f._rootNode, err = f.mkdir(absRoot, f.root)
|
||||
return f._rootNode, err
|
||||
}
|
||||
|
||||
@@ -437,7 +433,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))
|
||||
deleteErr := f.pacer.Call(func() (bool, error) {
|
||||
err := f.srv.Delete(item, true)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if deleteErr != nil {
|
||||
err = deleteErr
|
||||
@@ -451,7 +447,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.Node) (fs.Object, error) {
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *mega.Node) (fs.Object, error) {
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
@@ -461,7 +457,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.No
|
||||
// Set info
|
||||
err = o.setMetaData(info)
|
||||
} else {
|
||||
err = o.readMetaData(ctx) // reads info and meta, returning an error
|
||||
err = o.readMetaData() // reads info and meta, returning an error
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -472,7 +468,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.No
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
return f.newObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// list the objects into the function supplied
|
||||
@@ -510,7 +506,7 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
dirNode, err := f.lookupDir(ctx, dir)
|
||||
dirNode, err := f.lookupDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -522,7 +518,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash())
|
||||
entries = append(entries, d)
|
||||
case mega.FILE:
|
||||
o, err := f.newObjectWithInfo(ctx, remote, info)
|
||||
o, err := f.newObjectWithInfo(remote, info)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
@@ -546,8 +542,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Returns the dirNode, object, leaf and error
|
||||
//
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
|
||||
dirNode, leaf, err = f.mkdirParent(ctx, remote)
|
||||
func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
|
||||
dirNode, leaf, err = f.mkdirParent(remote)
|
||||
if err != nil {
|
||||
return nil, nil, leaf, err
|
||||
}
|
||||
@@ -569,7 +565,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
|
||||
// This will create a duplicate if we upload a new file without
|
||||
// checking to see if there is one already - use Put() for that.
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
|
||||
existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
@@ -595,7 +591,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
size := src.Size()
|
||||
modTime := src.ModTime(ctx)
|
||||
|
||||
o, _, _, err := f.createObject(ctx, remote, modTime, size)
|
||||
o, _, _, err := f.createObject(remote, modTime, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -604,30 +600,30 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
|
||||
// Mkdir creates the directory if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
rootNode, err := f.findRoot(ctx, true)
|
||||
rootNode, err := f.findRoot(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.mkdir(ctx, rootNode, dir)
|
||||
_, err = f.mkdir(rootNode, dir)
|
||||
return errors.Wrap(err, "Mkdir failed")
|
||||
}
|
||||
|
||||
// deleteNode removes a file or directory, observing useTrash
|
||||
func (f *Fs) deleteNode(ctx context.Context, node *mega.Node) (err error) {
|
||||
func (f *Fs) deleteNode(node *mega.Node) (err error) {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = f.srv.Delete(node, f.opt.HardDelete)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// purgeCheck removes the directory dir, if check is set then it
|
||||
// refuses to do so if it has anything in
|
||||
func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
func (f *Fs) purgeCheck(dir string, check bool) error {
|
||||
f.mkdirMu.Lock()
|
||||
defer f.mkdirMu.Unlock()
|
||||
|
||||
rootNode, err := f.findRoot(ctx, false)
|
||||
rootNode, err := f.findRoot(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -648,7 +644,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
waitEvent := f.srv.WaitEventsStart()
|
||||
|
||||
err = f.deleteNode(ctx, dirNode)
|
||||
err = f.deleteNode(dirNode)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "delete directory node failed")
|
||||
}
|
||||
@@ -666,7 +662,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
//
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.purgeCheck(ctx, dir, true)
|
||||
return f.purgeCheck(dir, true)
|
||||
}
|
||||
|
||||
// Precision return the precision of this Fs
|
||||
@@ -680,13 +676,13 @@ func (f *Fs) Precision() time.Duration {
|
||||
// deleting all the files quicker than just running Remove() on the
|
||||
// result of List()
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.purgeCheck(ctx, dir, false)
|
||||
return f.purgeCheck(dir, false)
|
||||
}
|
||||
|
||||
// move a file or folder (srcFs, srcRemote, info) to (f, dstRemote)
|
||||
//
|
||||
// info will be updates
|
||||
func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
|
||||
func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node) (err error) {
|
||||
var (
|
||||
dstFs = f
|
||||
srcDirNode, dstDirNode *mega.Node
|
||||
@@ -696,12 +692,12 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
|
||||
if dstRemote != "" {
|
||||
// lookup or create the destination parent directory
|
||||
dstDirNode, dstLeaf, err = dstFs.mkdirParent(ctx, dstRemote)
|
||||
dstDirNode, dstLeaf, err = dstFs.mkdirParent(dstRemote)
|
||||
} else {
|
||||
// find or create the parent of the root directory
|
||||
absRoot := dstFs.srv.FS.GetRoot()
|
||||
dstParent, dstLeaf = path.Split(dstFs.root)
|
||||
dstDirNode, err = dstFs.mkdir(ctx, absRoot, dstParent)
|
||||
dstDirNode, err = dstFs.mkdir(absRoot, dstParent)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "server-side move failed to make dst parent dir")
|
||||
@@ -709,7 +705,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
|
||||
if srcRemote != "" {
|
||||
// lookup the existing parent directory
|
||||
srcDirNode, srcLeaf, err = srcFs.lookupParentDir(ctx, srcRemote)
|
||||
srcDirNode, srcLeaf, err = srcFs.lookupParentDir(srcRemote)
|
||||
} else {
|
||||
// lookup the existing root parent
|
||||
absRoot := srcFs.srv.FS.GetRoot()
|
||||
@@ -725,7 +721,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
//log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName())
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = f.srv.Move(info, dstDirNode)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "server-side move failed")
|
||||
@@ -739,7 +735,7 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = f.srv.Rename(info, f.opt.Enc.FromStandardName(dstLeaf))
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "server-side rename failed")
|
||||
@@ -771,7 +767,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
|
||||
// Do the move
|
||||
err := f.move(ctx, remote, srcObj.fs, srcObj.remote, srcObj.info)
|
||||
err := f.move(remote, srcObj.fs, srcObj.remote, srcObj.info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -802,13 +798,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// find the source
|
||||
info, err := srcFs.lookupDir(ctx, srcRemote)
|
||||
info, err := srcFs.lookupDir(srcRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check the destination doesn't exist
|
||||
_, err = dstFs.lookupDir(ctx, dstRemote)
|
||||
_, err = dstFs.lookupDir(dstRemote)
|
||||
if err == nil {
|
||||
return fs.ErrorDirExists
|
||||
} else if err != fs.ErrorDirNotFound {
|
||||
@@ -816,7 +812,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// Do the move
|
||||
err = f.move(ctx, dstRemote, srcFs, srcRemote, info)
|
||||
err = f.move(dstRemote, srcFs, srcRemote, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -842,7 +838,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
root, err := f.findRoot(ctx, false)
|
||||
root, err := f.findRoot(false)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "PublicLink failed to find root node")
|
||||
}
|
||||
@@ -890,7 +886,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
fs.Infof(srcDir, "merging %q", f.opt.Enc.ToStandardName(info.GetName()))
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = f.srv.Move(info, dstDirNode)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "MergeDirs move failed on %q in %v", f.opt.Enc.ToStandardName(info.GetName()), srcDir)
|
||||
@@ -898,7 +894,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
}
|
||||
// rmdir (into trash) the now empty source directory
|
||||
fs.Infof(srcDir, "removing empty directory")
|
||||
err = f.deleteNode(ctx, srcDirNode)
|
||||
err = f.deleteNode(srcDirNode)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
|
||||
}
|
||||
@@ -912,7 +908,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
q, err = f.srv.GetQuota()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get Mega Quota")
|
||||
@@ -967,11 +963,11 @@ func (o *Object) setMetaData(info *mega.Node) (err error) {
|
||||
// readMetaData gets the metadata if it hasn't already been fetched
|
||||
//
|
||||
// it also sets the info
|
||||
func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
func (o *Object) readMetaData() (err error) {
|
||||
if o.info != nil {
|
||||
return nil
|
||||
}
|
||||
info, err := o.fs.readMetaDataForPath(ctx, o.remote)
|
||||
info, err := o.fs.readMetaDataForPath(o.remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
err = fs.ErrorObjectNotFound
|
||||
@@ -1002,7 +998,6 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// openObject represents a download in progress
|
||||
type openObject struct {
|
||||
ctx context.Context
|
||||
mu sync.Mutex
|
||||
o *Object
|
||||
d *mega.Download
|
||||
@@ -1013,14 +1008,14 @@ type openObject struct {
|
||||
}
|
||||
|
||||
// get the next chunk
|
||||
func (oo *openObject) getChunk(ctx context.Context) (err error) {
|
||||
func (oo *openObject) getChunk() (err error) {
|
||||
if oo.id >= oo.d.Chunks() {
|
||||
return io.EOF
|
||||
}
|
||||
var chunk []byte
|
||||
err = oo.o.fs.pacer.Call(func() (bool, error) {
|
||||
chunk, err = oo.d.DownloadChunk(oo.id)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1050,7 +1045,7 @@ func (oo *openObject) Read(p []byte) (n int, err error) {
|
||||
oo.skip -= int64(size)
|
||||
}
|
||||
if len(oo.chunk) == 0 {
|
||||
err = oo.getChunk(oo.ctx)
|
||||
err = oo.getChunk()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -1073,7 +1068,7 @@ func (oo *openObject) Close() (err error) {
|
||||
}
|
||||
err = oo.o.fs.pacer.Call(func() (bool, error) {
|
||||
err = oo.d.Finish()
|
||||
return shouldRetry(oo.ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to finish download")
|
||||
@@ -1101,14 +1096,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var d *mega.Download
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
d, err = o.fs.srv.NewDownload(o.info)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "open download file failed")
|
||||
}
|
||||
|
||||
oo := &openObject{
|
||||
ctx: ctx,
|
||||
o: o,
|
||||
d: d,
|
||||
skip: offset,
|
||||
@@ -1131,7 +1125,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remote := o.Remote()
|
||||
|
||||
// Create the parent directory
|
||||
dirNode, leaf, err := o.fs.mkdirParent(ctx, remote)
|
||||
dirNode, leaf, err := o.fs.mkdirParent(remote)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "update make parent dir failed")
|
||||
}
|
||||
@@ -1139,7 +1133,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var u *mega.Upload
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
u, err = o.fs.srv.NewUpload(dirNode, o.fs.opt.Enc.FromStandardName(leaf), size)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload file failed to create session")
|
||||
@@ -1160,7 +1154,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = u.UploadChunk(id, chunk)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload file failed to upload chunk")
|
||||
@@ -1171,7 +1165,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var info *mega.Node
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
info, err = u.Finish()
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to finish upload")
|
||||
@@ -1179,7 +1173,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// If the upload succeeded and the original object existed, then delete it
|
||||
if o.info != nil {
|
||||
err = o.fs.deleteNode(ctx, o.info)
|
||||
err = o.fs.deleteNode(o.info)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload failed to remove old version")
|
||||
}
|
||||
@@ -1191,7 +1185,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
err := o.fs.deleteNode(ctx, o.info)
|
||||
err := o.fs.deleteNode(o.info)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Remove object failed")
|
||||
}
|
||||
|
||||
@@ -302,6 +302,7 @@ func init() {
|
||||
|
||||
m.Set(configDriveID, finalDriveID)
|
||||
m.Set(configDriveType, rootItem.ParentReference.DriveType)
|
||||
config.SaveConfig()
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "region",
|
||||
@@ -549,10 +550,7 @@ var errAsyncJobAccessDenied = errors.New("async job failed - access denied")
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
retry := false
|
||||
if resp != nil {
|
||||
switch resp.StatusCode {
|
||||
@@ -599,7 +597,7 @@ func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID s
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
return info, resp, err
|
||||
@@ -615,7 +613,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
|
||||
opts.Path = strings.TrimSuffix(opts.Path, ":")
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return info, resp, err
|
||||
}
|
||||
@@ -871,7 +869,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -903,7 +901,7 @@ OUTER:
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -1040,7 +1038,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1196,7 +1194,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, ©Req, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1289,7 +1287,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var info api.Item
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1356,7 +1354,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
var info api.Item
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1382,7 +1380,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "about failed")
|
||||
@@ -1432,7 +1430,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var result api.CreateShareLinkResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &share, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
@@ -1477,7 +1475,7 @@ func (o *Object) deleteVersions(ctx context.Context) error {
|
||||
var versions api.VersionsResponse
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1504,7 +1502,7 @@ func (o *Object) deleteVersion(ctx context.Context, ID string) error {
|
||||
opts.NoResponse = true
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1655,7 +1653,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
|
||||
var info *api.Item
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
// Remove versions if required
|
||||
if o.fs.opt.NoVersions {
|
||||
@@ -1697,7 +1695,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1725,7 +1723,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
|
||||
err = errors.New(err.Error() + " (is it a OneNote file?)")
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return response, err
|
||||
}
|
||||
@@ -1740,7 +1738,7 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -1800,11 +1798,11 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
|
||||
return true, errors.Wrapf(err, "retry this chunk skipping %d bytes", skip)
|
||||
}
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
body, err = rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
if resp.StatusCode == 200 || resp.StatusCode == 201 {
|
||||
// we are done :)
|
||||
@@ -1827,7 +1825,7 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -1898,7 +1896,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
|
||||
err = errors.New(err.Error() + " (is it a OneNote file?)")
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -119,7 +119,6 @@ type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
id string // ID of the file
|
||||
parent string // ID of the parent directory
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
@@ -207,7 +206,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
Path: "/session/login.json",
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create session")
|
||||
@@ -234,7 +233,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.newObjectWithInfo(ctx, remote, nil, "")
|
||||
_, err := tempF.newObjectWithInfo(ctx, remote, nil)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
@@ -294,7 +293,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
Path: "/folder/remove.json",
|
||||
}
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -389,7 +388,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
Path: "/file/move_copy.json",
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -446,7 +445,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
Path: "/file/move_copy.json",
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -495,7 +494,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
Path: "/folder/move_copy.json",
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(src, "DirMove error %v", err)
|
||||
@@ -518,7 +517,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, parent string) (fs.Object, error) {
|
||||
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File) (fs.Object, error) {
|
||||
// fs.Debugf(nil, "newObjectWithInfo(%s, %v)", remote, file)
|
||||
|
||||
var o *Object
|
||||
@@ -527,7 +526,6 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, p
|
||||
fs: f,
|
||||
remote: remote,
|
||||
id: file.FileID,
|
||||
parent: parent,
|
||||
modTime: time.Unix(file.DateModified, 0),
|
||||
size: file.Size,
|
||||
md5: file.FileHash,
|
||||
@@ -550,7 +548,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *File, p
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
// fs.Debugf(nil, "NewObject(\"%s\")", remote)
|
||||
return f.newObjectWithInfo(ctx, remote, nil, "")
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
}
|
||||
|
||||
// Creates from the parameters passed in a half finished Object which
|
||||
@@ -583,7 +581,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -633,7 +631,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
Path: "/upload/create_file.json",
|
||||
}
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create file")
|
||||
@@ -659,10 +657,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -688,7 +683,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
Path: "/folder.json",
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -716,7 +711,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID,
|
||||
}
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", false, errors.Wrap(err, "failed to get folder list")
|
||||
@@ -759,7 +754,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
folderList := FolderList{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get folder list")
|
||||
@@ -773,7 +768,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
f.dirCache.Put(remote, folder.FolderID)
|
||||
d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
|
||||
d.SetItems(int64(folder.ChildFolders))
|
||||
d.SetParentID(directoryID)
|
||||
entries = append(entries, d)
|
||||
}
|
||||
|
||||
@@ -781,7 +775,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
file.Name = f.opt.Enc.ToStandardName(file.Name)
|
||||
// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
|
||||
remote := path.Join(dir, file.Name)
|
||||
o, err := f.newObjectWithInfo(ctx, remote, &file, directoryID)
|
||||
o, err := f.newObjectWithInfo(ctx, remote, &file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -848,7 +842,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
}
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
o.modTime = modTime
|
||||
@@ -868,7 +862,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open file)")
|
||||
@@ -887,7 +881,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id,
|
||||
}
|
||||
resp, err := o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -916,7 +910,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Path: "/upload/open_file_upload.json",
|
||||
}
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create file")
|
||||
@@ -960,7 +954,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
}
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create file")
|
||||
@@ -983,7 +977,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Path: "/upload/close_file_upload.json",
|
||||
}
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create file")
|
||||
@@ -1009,7 +1003,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Path: "/file/access.json",
|
||||
}
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1035,7 +1029,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
|
||||
}
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get folder list")
|
||||
@@ -1059,11 +1053,6 @@ func (o *Object) ID() string {
|
||||
return o.id
|
||||
}
|
||||
|
||||
// ParentID returns the ID of the Object parent directory if known, or "" if not
|
||||
func (o *Object) ParentID() string {
|
||||
return o.parent
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
@@ -1074,5 +1063,4 @@ var (
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
_ fs.ParentIDer = (*Object)(nil)
|
||||
)
|
||||
|
||||
@@ -213,10 +213,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
doRetry := false
|
||||
|
||||
// Check if it is an api.Error
|
||||
@@ -408,7 +405,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -463,7 +460,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -600,7 +597,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rmdir failed")
|
||||
@@ -665,7 +662,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -703,7 +700,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -743,7 +740,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -790,7 +787,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -817,7 +814,7 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -841,7 +838,7 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -872,7 +869,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
|
||||
err = q.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "about failed")
|
||||
@@ -930,7 +927,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1049,7 +1046,7 @@ func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1075,7 +1072,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1137,7 +1134,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// opts.Body=0), so upload it as a multipart form POST with
|
||||
// Content-Length set.
|
||||
if size == 0 {
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, opts.Parameters, "content", leaf)
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to make multipart upload for 0 length file")
|
||||
}
|
||||
@@ -1154,7 +1151,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// sometimes pcloud leaves a half complete file on
|
||||
@@ -1184,7 +1181,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -176,10 +176,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -373,7 +370,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -410,7 +407,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -584,7 +581,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
var result api.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rmdir failed")
|
||||
@@ -663,7 +660,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
|
||||
var result api.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Move http")
|
||||
@@ -772,7 +769,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "CreateDir http")
|
||||
@@ -899,7 +896,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -937,7 +934,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
// Just check the download URL resolves - sometimes
|
||||
// the URLs returned by premiumize.me don't resolve so
|
||||
@@ -996,7 +993,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var result api.Response
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload file http")
|
||||
@@ -1038,7 +1035,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
|
||||
var result api.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rename http")
|
||||
@@ -1063,7 +1060,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
|
||||
var result api.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "remove http")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package putio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
@@ -30,10 +29,7 @@ func (e *statusCodeError) Temporary() bool {
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(err error) (bool, error) {
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -147,7 +147,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
|
||||
entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return itoa(entry.ID), err
|
||||
}
|
||||
@@ -164,7 +164,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "listing file: %d", fileID)
|
||||
children, _, err = f.client.Files.List(ctx, fileID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
|
||||
@@ -205,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "listing files inside List: %d", parentID)
|
||||
children, _, err = f.client.Files.List(ctx, parentID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -271,7 +271,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "getting file: %d", fileID)
|
||||
entry, err = f.client.Files.Get(ctx, fileID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -295,7 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
|
||||
req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
|
||||
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||
resp, err := f.oAuthClient.Do(req)
|
||||
retry, err := shouldRetry(ctx, err)
|
||||
retry, err := shouldRetry(err)
|
||||
if retry {
|
||||
return true, err
|
||||
}
|
||||
@@ -320,7 +320,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(f, "Sending zero length chunk")
|
||||
_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -344,13 +344,13 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
// Get file offset and seek to the position
|
||||
offset, err := f.getServerOffset(ctx, location)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
}
|
||||
sentBytes := offset - chunkStart
|
||||
fs.Debugf(f, "sentBytes: %d", sentBytes)
|
||||
_, err = chunk.Seek(sentBytes, io.SeekStart)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
}
|
||||
transferOffset = offset
|
||||
reqSize = chunkSize - sentBytes
|
||||
@@ -367,7 +367,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
offsetMismatch = true
|
||||
return true, errors.New("connection broken")
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -479,7 +479,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "listing files: %d", dirID)
|
||||
children, _, err = f.client.Files.List(ctx, dirID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Rmdir")
|
||||
@@ -493,7 +493,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "deleting file: %d", dirID)
|
||||
err = f.client.Files.Delete(ctx, dirID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
f.dirCache.FlushDir(dir)
|
||||
return err
|
||||
@@ -552,7 +552,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
// fs.Debugf(f, "copying file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
|
||||
_, err = f.client.Do(req, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -591,7 +591,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
// fs.Debugf(f, "moving file (%d) to parent_id: %s", srcObj.file.ID, directoryID)
|
||||
_, err = f.client.Do(req, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -631,7 +631,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
// fs.Debugf(f, "moving file (%s) to parent_id: %s", srcID, dstDirectoryID)
|
||||
_, err = f.client.Do(req, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return err
|
||||
@@ -644,7 +644,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "getting account info")
|
||||
ai, err = f.client.Account.Info(ctx)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "about failed")
|
||||
@@ -678,6 +678,6 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
}
|
||||
// fs.Debugf(f, "emptying trash")
|
||||
_, err = f.client.Do(req, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
|
||||
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode == 404 {
|
||||
return false, fs.ErrorObjectNotFound
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -220,7 +220,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var storageURL string
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
storageURL, err = o.fs.client.Files.URL(ctx, o.file.ID, true)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
@@ -231,7 +231,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, storageURL, nil)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", o.fs.client.UserAgent)
|
||||
|
||||
@@ -241,7 +241,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
// fs.Debugf(o, "opening file: id=%d", o.file.ID)
|
||||
resp, err = o.fs.httpClient.Do(req)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
|
||||
_ = resp.Body.Close()
|
||||
@@ -283,6 +283,6 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(o, "removing file: id=%d", o.file.ID)
|
||||
err = o.fs.client.Files.Delete(ctx, o.file.ID)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1399,10 +1399,7 @@ var retryErrorCodes = []int{
|
||||
//S3 is pretty resilient, and the built in retry handling is probably sufficient
|
||||
// as it should notice closed connections and timeouts which are the most likely
|
||||
// sort of failure modes
|
||||
func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(err error) (bool, error) {
|
||||
// If this is an awserr object, try and extract more useful information to determine if we should retry
|
||||
if awsError, ok := err.(awserr.Error); ok {
|
||||
// Simple case, check the original embedded error in case it's generically retryable
|
||||
@@ -1414,7 +1411,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
// 301 if wrong region for bucket - can only update if running from a bucket
|
||||
if f.rootBucket != "" {
|
||||
if reqErr.StatusCode() == http.StatusMovedPermanently {
|
||||
urfbErr := f.updateRegionForBucket(ctx, f.rootBucket)
|
||||
urfbErr := f.updateRegionForBucket(f.rootBucket)
|
||||
if urfbErr != nil {
|
||||
fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
|
||||
return false, err
|
||||
@@ -1562,8 +1559,6 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
|
||||
// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
|
||||
awsSessionOpts.SharedConfigState = session.SharedConfigEnable
|
||||
// Set the name of the profile if supplied
|
||||
awsSessionOpts.Profile = opt.Profile
|
||||
}
|
||||
ses, err := session.NewSessionWithOptions(awsSessionOpts)
|
||||
if err != nil {
|
||||
@@ -1746,7 +1741,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
// Gets the bucket location
|
||||
func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, error) {
|
||||
func (f *Fs) getBucketLocation(bucket string) (string, error) {
|
||||
req := s3.GetBucketLocationInput{
|
||||
Bucket: &bucket,
|
||||
}
|
||||
@@ -1754,7 +1749,7 @@ func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, erro
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.c.GetBucketLocation(&req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1764,8 +1759,8 @@ func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, erro
|
||||
|
||||
// Updates the region for the bucket by reading the region from the
|
||||
// bucket then updating the session.
|
||||
func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
|
||||
region, err := f.getBucketLocation(ctx, bucket)
|
||||
func (f *Fs) updateRegionForBucket(bucket string) error {
|
||||
region, err := f.getBucketLocation(bucket)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "reading bucket location failed")
|
||||
}
|
||||
@@ -1859,7 +1854,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
}
|
||||
}
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if awsErr, ok := err.(awserr.RequestFailure); ok {
|
||||
@@ -2006,7 +2001,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
|
||||
var resp *s3.ListBucketsOutput
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.c.ListBucketsWithContext(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2121,7 +2116,7 @@ func (f *Fs) bucketExists(ctx context.Context, bucket string) (bool, error) {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.HeadBucketWithContext(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
@@ -2157,7 +2152,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.CreateBucketWithContext(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
fs.Infof(f, "Bucket %q created with ACL %q", bucket, f.opt.BucketACL)
|
||||
@@ -2187,7 +2182,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteBucketWithContext(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
fs.Infof(f, "Bucket %q deleted", bucket)
|
||||
@@ -2247,7 +2242,7 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.CopyObjectWithContext(ctx, req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2291,7 +2286,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
|
||||
if err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
cout, err = f.c.CreateMultipartUploadWithContext(ctx, req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2307,7 +2302,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
})()
|
||||
|
||||
@@ -2330,7 +2325,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
|
||||
uploadPartReq.CopySourceRange = aws.String(calculateRange(partSize, partNum-1, numParts, srcSize))
|
||||
uout, err := f.c.UploadPartCopyWithContext(ctx, uploadPartReq)
|
||||
if err != nil {
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
}
|
||||
parts = append(parts, &s3.CompletedPart{
|
||||
PartNumber: &partNum,
|
||||
@@ -2352,7 +2347,7 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst
|
||||
RequestPayer: req.RequestPayer,
|
||||
UploadId: uid,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2583,7 +2578,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
reqCopy.Key = &bucketPath
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.c.RestoreObject(&reqCopy)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
st.Status = err.Error()
|
||||
@@ -2631,7 +2626,7 @@ func (f *Fs) listMultipartUploads(ctx context.Context, bucket, key string) (uplo
|
||||
var resp *s3.ListMultipartUploadsOutput
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.c.ListMultipartUploads(&req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "list multipart uploads bucket %q key %q", bucket, key)
|
||||
@@ -2806,7 +2801,7 @@ func (o *Object) headObject(ctx context.Context) (resp *s3.HeadObjectOutput, err
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.c.HeadObjectWithContext(ctx, &req)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if awsErr, ok := err.(awserr.RequestFailure); ok {
|
||||
@@ -2962,7 +2957,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var err error
|
||||
httpReq.HTTPRequest = httpReq.HTTPRequest.WithContext(ctx)
|
||||
err = httpReq.Send()
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err, ok := err.(awserr.RequestFailure); ok {
|
||||
if err.Code() == "InvalidObjectState" {
|
||||
@@ -3021,7 +3016,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
cout, err = f.c.CreateMultipartUploadWithContext(ctx, &mReq)
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "multipart upload failed to initialise")
|
||||
@@ -3040,7 +3035,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if errCancel != nil {
|
||||
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
|
||||
@@ -3116,7 +3111,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
|
||||
if err != nil {
|
||||
if partNum <= int64(concurrency) {
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
}
|
||||
// retry all chunks once have done the first batch
|
||||
return true, err
|
||||
@@ -3156,7 +3151,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
RequestPayer: req.RequestPayer,
|
||||
UploadId: uid,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "multipart upload failed to finalise")
|
||||
@@ -3311,11 +3306,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var err error
|
||||
resp, err = o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
}
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
|
||||
return false, nil
|
||||
@@ -3366,7 +3361,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
}
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := o.fs.c.DeleteObjectWithContext(ctx, &req)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -372,6 +372,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
m.Set(configAuthToken, token)
|
||||
// And delete any previous entry for password
|
||||
m.Set(configPassword, "")
|
||||
config.SaveConfig()
|
||||
// And we're done here
|
||||
break
|
||||
}
|
||||
@@ -407,10 +408,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// For 429 errors look at the Retry-After: header and
|
||||
// set the retry appropriately, starting with a minimum of 1
|
||||
// second if it isn't set.
|
||||
|
||||
@@ -86,7 +86,7 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -112,7 +112,7 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -139,7 +139,7 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -170,7 +170,7 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -197,7 +197,7 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -228,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -271,7 +271,7 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -316,7 +316,7 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -358,7 +358,7 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -398,7 +398,7 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -438,7 +438,7 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -474,7 +474,7 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -505,7 +505,7 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -539,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete file")
|
||||
@@ -565,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -614,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -659,7 +659,7 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -682,7 +682,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
|
||||
"need_idx_progress": {"true"},
|
||||
"replace": {"1"},
|
||||
}
|
||||
formReader, contentType, _, err := rest.MultipartUpload(ctx, in, parameters, "file", f.opt.Enc.FromStandardName(filename))
|
||||
formReader, contentType, _, err := rest.MultipartUpload(in, parameters, "file", f.opt.Enc.FromStandardName(filename))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make multipart upload")
|
||||
}
|
||||
@@ -739,7 +739,7 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -777,7 +777,7 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -818,7 +818,7 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -860,7 +860,7 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -900,7 +900,7 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -938,7 +938,7 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -976,7 +976,7 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -1030,7 +1030,7 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -1075,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
|
||||
@@ -204,25 +204,6 @@ Fstat instead of Stat which is called on an already open file handle.
|
||||
It has been found that this helps with IBM Sterling SFTP servers which have
|
||||
"extractability" level set to 1 which means only 1 file can be opened at
|
||||
any given time.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_concurrent_reads",
|
||||
Default: false,
|
||||
Help: `If set don't use concurrent reads
|
||||
|
||||
Normally concurrent reads are safe to use and not using them will
|
||||
degrade performance, so this option is disabled by default.
|
||||
|
||||
Some servers limit the amount number of times a file can be
|
||||
downloaded. Using concurrent reads can trigger this limit, so if you
|
||||
have a server which returns
|
||||
|
||||
Failed to copy: file does not exist
|
||||
|
||||
Then you may need to enable this flag.
|
||||
|
||||
If concurrent reads are disabled, the use_fstat option is ignored.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -243,29 +224,28 @@ Set to 0 to keep connections indefinitely.
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyPem string `config:"key_pem"`
|
||||
KeyFile string `config:"key_file"`
|
||||
KeyFilePass string `config:"key_file_pass"`
|
||||
PubKeyFile string `config:"pubkey_file"`
|
||||
KnownHostsFile string `config:"known_hosts_file"`
|
||||
KeyUseAgent bool `config:"key_use_agent"`
|
||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
UseFstat bool `config:"use_fstat"`
|
||||
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyPem string `config:"key_pem"`
|
||||
KeyFile string `config:"key_file"`
|
||||
KeyFilePass string `config:"key_file_pass"`
|
||||
PubKeyFile string `config:"pubkey_file"`
|
||||
KnownHostsFile string `config:"known_hosts_file"`
|
||||
KeyUseAgent bool `config:"key_use_agent"`
|
||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
UseFstat bool `config:"use_fstat"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote SFTP files
|
||||
@@ -393,10 +373,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
|
||||
}
|
||||
}
|
||||
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
|
||||
opts = append(opts,
|
||||
sftp.UseFstat(f.opt.UseFstat),
|
||||
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
|
||||
)
|
||||
opts = append(opts, sftp.UseFstat(f.opt.UseFstat))
|
||||
|
||||
return sftp.NewClientPipe(pr, pw, opts...)
|
||||
}
|
||||
@@ -1354,19 +1331,18 @@ func (o *Object) stat(ctx context.Context) error {
|
||||
//
|
||||
// it also updates the info field
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
if !o.fs.opt.SetModTime {
|
||||
return nil
|
||||
if o.fs.opt.SetModTime {
|
||||
c, err := o.fs.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime")
|
||||
}
|
||||
err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
|
||||
o.fs.putSftpConnection(&c, err)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime failed")
|
||||
}
|
||||
}
|
||||
c, err := o.fs.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime")
|
||||
}
|
||||
err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
|
||||
o.fs.putSftpConnection(&c, err)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime failed")
|
||||
}
|
||||
err = o.stat(ctx)
|
||||
err := o.stat(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime stat failed")
|
||||
}
|
||||
@@ -1497,28 +1473,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remove()
|
||||
return errors.Wrap(err, "Update Close failed")
|
||||
}
|
||||
|
||||
// Set the mod time - this stats the object if o.fs.opt.SetModTime == true
|
||||
err = o.SetModTime(ctx, src.ModTime(ctx))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Update SetModTime failed")
|
||||
}
|
||||
|
||||
// Stat the file after the upload to read its stats back if o.fs.opt.SetModTime == false
|
||||
if !o.fs.opt.SetModTime {
|
||||
err = o.stat(ctx)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// In the specific case of o.fs.opt.SetModTime == false
|
||||
// if the object wasn't found then don't return an error
|
||||
fs.Debugf(o, "Not found after upload with set_modtime=false so returning best guess")
|
||||
o.modTime = src.ModTime(ctx)
|
||||
o.size = src.Size()
|
||||
o.mode = os.FileMode(0666) // regular file
|
||||
} else if err != nil {
|
||||
return errors.Wrap(err, "Update stat failed")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -299,10 +299,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -327,7 +324,7 @@ func (f *Fs) readMetaDataForIDPath(ctx context.Context, id, path string, directo
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &item)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusNotFound {
|
||||
@@ -634,7 +631,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &req, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "CreateDir")
|
||||
@@ -666,7 +663,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -915,7 +912,7 @@ func (f *Fs) updateItem(ctx context.Context, id, leaf, directoryID string, modTi
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &update, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1136,7 +1133,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
var info *api.Item
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1297,7 +1294,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
var dl api.DownloadSpecification
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "open: fetch download specification")
|
||||
@@ -1312,7 +1309,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "open")
|
||||
@@ -1368,7 +1365,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload get specification")
|
||||
@@ -1393,7 +1390,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var finish api.UploadFinishResponse
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &finish)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "upload file")
|
||||
@@ -1429,7 +1426,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "remove")
|
||||
|
||||
@@ -155,7 +155,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
|
||||
err := up.f.pacer.Call(func() (bool, error) {
|
||||
resp, err := up.f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
}
|
||||
respBody, err = rest.ReadBody(resp)
|
||||
// retry all errors now that the multipart upload has started
|
||||
|
||||
@@ -111,7 +111,7 @@ func init() {
|
||||
// FIXME
|
||||
//err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil)
|
||||
// return shouldRetry(ctx, resp, err)
|
||||
// return shouldRetry(resp, err)
|
||||
//})
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get token: %v", err)
|
||||
@@ -248,10 +248,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -291,7 +288,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode == http.StatusNotFound {
|
||||
@@ -328,7 +325,7 @@ func (f *Fs) getAuthToken(ctx context.Context) error {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get authorization")
|
||||
@@ -376,7 +373,7 @@ func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &user)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get user")
|
||||
@@ -570,7 +567,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -621,7 +618,7 @@ OUTER:
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -777,7 +774,7 @@ func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string,
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
// Move file/dir to deleted files if not hard delete
|
||||
@@ -883,7 +880,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, ©File, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -937,7 +934,7 @@ func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, &move, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -967,7 +964,7 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err
|
||||
var resp *http.Response
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, &move, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1056,7 +1053,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var info *api.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1185,7 +1182,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1207,7 +1204,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1265,7 +1262,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload file")
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -292,10 +291,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(err error) (bool, error) {
|
||||
// If this is a swift.Error object extract the HTTP error code
|
||||
if swiftError, ok := err.(*swift.Error); ok {
|
||||
for _, e := range retryErrorCodes {
|
||||
@@ -311,7 +307,7 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
// shouldRetryHeaders returns a boolean as to whether this err
|
||||
// deserves to be retried. It reads the headers passed in looking for
|
||||
// `Retry-After`. It returns the err as a convenience
|
||||
func shouldRetryHeaders(ctx context.Context, headers swift.Headers, err error) (bool, error) {
|
||||
func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
|
||||
if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
|
||||
if value := headers["Retry-After"]; value != "" {
|
||||
retryAfter, parseErr := strconv.Atoi(value)
|
||||
@@ -330,7 +326,7 @@ func shouldRetryHeaders(ctx context.Context, headers swift.Headers, err error) (
|
||||
}
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
}
|
||||
|
||||
// parsePath parses a remote 'url'
|
||||
@@ -472,7 +468,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
info, rxHeaders, err = f.c.Object(ctx, f.rootContainer, encodedDirectory)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
if err == nil && info.ContentType != directoryMarkerContentType {
|
||||
newRoot := path.Dir(f.root)
|
||||
@@ -580,7 +576,7 @@ func (f *Fs) listContainerRoot(ctx context.Context, container, directory, prefix
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
objects, err = f.c.Objects(ctx, container, opts)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
for i := range objects {
|
||||
@@ -665,7 +661,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err
|
||||
var containers []swift.Container
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
containers, err = f.c.ContainersAll(ctx, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "container listing failed")
|
||||
@@ -757,7 +753,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
containers, err = f.c.ContainersAll(ctx, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "container listing failed")
|
||||
@@ -809,7 +805,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
_, rxHeaders, err = f.c.Container(ctx, container)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
}
|
||||
if err == swift.ContainerNotFound {
|
||||
@@ -819,7 +815,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = f.c.ContainerCreate(ctx, container, headers)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
fs.Infof(f, "Container %q created", container)
|
||||
@@ -840,7 +836,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
err := f.cache.Remove(container, func() error {
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
err := f.c.ContainerDelete(ctx, container)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
fs.Infof(f, "Container %q removed", container)
|
||||
@@ -906,125 +902,18 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
isLargeObject, err := srcObj.isLargeObject(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if isLargeObject {
|
||||
/*handle large object*/
|
||||
err = copyLargeObject(ctx, f, srcObj, dstContainer, dstPath)
|
||||
} else {
|
||||
srcContainer, srcPath := srcObj.split()
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
rxHeaders, err = f.c.ObjectCopy(ctx, srcContainer, srcPath, dstContainer, dstPath, nil)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
})
|
||||
}
|
||||
srcContainer, srcPath := srcObj.split()
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
rxHeaders, err = f.c.ObjectCopy(ctx, srcContainer, srcPath, dstContainer, dstPath, nil)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.NewObject(ctx, remote)
|
||||
}
|
||||
|
||||
func copyLargeObject(ctx context.Context, f *Fs, src *Object, dstContainer string, dstPath string) error {
|
||||
segmentsContainer := dstContainer + "_segments"
|
||||
err := f.makeContainer(ctx, segmentsContainer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
segments, err := src.getSegmentsLargeObject(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(segments) == 0 {
|
||||
return errors.New("could not copy object, list segments are empty")
|
||||
}
|
||||
nanoSeconds := time.Now().Nanosecond()
|
||||
prefixSegment := fmt.Sprintf("%v/%v/%s", nanoSeconds, src.size, strings.ReplaceAll(uuid.New().String(), "-", ""))
|
||||
copiedSegmentsLen := 10
|
||||
for _, value := range segments {
|
||||
if len(value) <= 0 {
|
||||
continue
|
||||
}
|
||||
fragment := value[0]
|
||||
if len(fragment) <= 0 {
|
||||
continue
|
||||
}
|
||||
copiedSegmentsLen = len(value)
|
||||
firstIndex := strings.Index(fragment, "/")
|
||||
if firstIndex < 0 {
|
||||
firstIndex = 0
|
||||
} else {
|
||||
firstIndex = firstIndex + 1
|
||||
}
|
||||
lastIndex := strings.LastIndex(fragment, "/")
|
||||
if lastIndex < 0 {
|
||||
lastIndex = len(fragment)
|
||||
} else {
|
||||
lastIndex = lastIndex - 1
|
||||
}
|
||||
prefixSegment = fragment[firstIndex:lastIndex]
|
||||
break
|
||||
}
|
||||
copiedSegments := make([]string, copiedSegmentsLen)
|
||||
defer handleCopyFail(ctx, f, segmentsContainer, copiedSegments, err)
|
||||
for c, ss := range segments {
|
||||
if len(ss) <= 0 {
|
||||
continue
|
||||
}
|
||||
for _, s := range ss {
|
||||
lastIndex := strings.LastIndex(s, "/")
|
||||
if lastIndex <= 0 {
|
||||
lastIndex = 0
|
||||
} else {
|
||||
lastIndex = lastIndex + 1
|
||||
}
|
||||
segmentName := dstPath + "/" + prefixSegment + "/" + s[lastIndex:]
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
rxHeaders, err = f.c.ObjectCopy(ctx, c, s, segmentsContainer, segmentName, nil)
|
||||
copiedSegments = append(copiedSegments, segmentName)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
m := swift.Metadata{}
|
||||
headers := m.ObjectHeaders()
|
||||
headers["X-Object-Manifest"] = urlEncode(fmt.Sprintf("%s/%s/%s", segmentsContainer, dstPath, prefixSegment))
|
||||
headers["Content-Length"] = "0"
|
||||
emptyReader := bytes.NewReader(nil)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
rxHeaders, err = f.c.ObjectPut(ctx, dstContainer, dstPath, emptyReader, true, "", src.contentType, headers)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
//remove copied segments when copy process failed
|
||||
func handleCopyFail(ctx context.Context, f *Fs, segmentsContainer string, segments []string, err error) {
|
||||
fs.Debugf(f, "handle copy segment fail")
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if len(segmentsContainer) == 0 {
|
||||
fs.Debugf(f, "invalid segments container")
|
||||
return
|
||||
}
|
||||
if len(segments) == 0 {
|
||||
fs.Debugf(f, "segments is empty")
|
||||
return
|
||||
}
|
||||
fs.Debugf(f, "action delete segments what copied")
|
||||
for _, v := range segments {
|
||||
_ = f.c.ObjectDelete(ctx, segmentsContainer, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.MD5)
|
||||
@@ -1152,7 +1041,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
container, containerPath := o.split()
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
info, h, err = o.fs.c.Object(ctx, container, containerPath)
|
||||
return shouldRetryHeaders(ctx, h, err)
|
||||
return shouldRetryHeaders(h, err)
|
||||
})
|
||||
if err != nil {
|
||||
if err == swift.ObjectNotFound {
|
||||
@@ -1211,7 +1100,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
container, containerPath := o.split()
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.c.ObjectUpdate(ctx, container, containerPath, newHeaders)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1232,7 +1121,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
in, rxHeaders, err = o.fs.c.ObjectOpen(ctx, container, containerPath, !isRanging, headers)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -1322,7 +1211,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
_, rxHeaders, err = o.fs.c.Container(ctx, segmentsContainer)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
if err == swift.ContainerNotFound {
|
||||
headers := swift.Headers{}
|
||||
@@ -1331,7 +1220,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.c.ContainerCreate(ctx, segmentsContainer, headers)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1352,8 +1241,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
|
||||
if segmentInfos == nil || len(segmentInfos) == 0 {
|
||||
return
|
||||
}
|
||||
_ctx := context.Background()
|
||||
deleteChunks(_ctx, o, segmentsContainer, segmentInfos)
|
||||
deleteChunks(ctx, o, segmentsContainer, segmentInfos)
|
||||
})()
|
||||
for {
|
||||
// can we read at least one byte?
|
||||
@@ -1379,7 +1267,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
|
||||
if err == nil {
|
||||
segmentInfos = append(segmentInfos, segmentPath)
|
||||
}
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1393,7 +1281,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift.
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, emptyReader, true, "", contentType, headers)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
@@ -1468,7 +1356,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var rxHeaders swift.Headers
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
rxHeaders, err = o.fs.c.ObjectPut(ctx, container, containerPath, in, true, "", contentType, headers)
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
return shouldRetryHeaders(rxHeaders, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1526,7 +1414,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
// Remove file/manifest first
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.c.ObjectDelete(ctx, container, containerPath)
|
||||
return shouldRetry(ctx, err)
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package swift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -33,7 +32,6 @@ func TestInternalUrlEncode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInternalShouldRetryHeaders(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
headers := swift.Headers{
|
||||
"Content-Length": "64",
|
||||
"Content-Type": "text/html; charset=UTF-8",
|
||||
@@ -47,7 +45,7 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
|
||||
|
||||
// Short sleep should just do the sleep
|
||||
start := time.Now()
|
||||
retry, gotErr := shouldRetryHeaders(ctx, headers, err)
|
||||
retry, gotErr := shouldRetryHeaders(headers, err)
|
||||
dt := time.Since(start)
|
||||
assert.True(t, retry)
|
||||
assert.Equal(t, err, gotErr)
|
||||
@@ -56,7 +54,7 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
|
||||
// Long sleep should return RetryError
|
||||
headers["Retry-After"] = "3600"
|
||||
start = time.Now()
|
||||
retry, gotErr = shouldRetryHeaders(ctx, headers, err)
|
||||
retry, gotErr = shouldRetryHeaders(headers, err)
|
||||
dt = time.Since(start)
|
||||
assert.True(t, dt < time.Second)
|
||||
assert.False(t, retry)
|
||||
|
||||
@@ -80,7 +80,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("NoChunk", f.testNoChunk)
|
||||
t.Run("WithChunk", f.testWithChunk)
|
||||
t.Run("WithChunkFail", f.testWithChunkFail)
|
||||
t.Run("CopyLargeObject", f.testCopyLargeObject)
|
||||
}
|
||||
|
||||
func (f *Fs) testWithChunk(t *testing.T) {
|
||||
@@ -155,39 +154,4 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
|
||||
require.Empty(t, objs)
|
||||
}
|
||||
|
||||
func (f *Fs) testCopyLargeObject(t *testing.T) {
|
||||
preConfChunkSize := f.opt.ChunkSize
|
||||
preConfChunk := f.opt.NoChunk
|
||||
f.opt.NoChunk = false
|
||||
f.opt.ChunkSize = 1024 * fs.Byte
|
||||
defer func() {
|
||||
//restore old config after test
|
||||
f.opt.ChunkSize = preConfChunkSize
|
||||
f.opt.NoChunk = preConfChunk
|
||||
}()
|
||||
|
||||
file := fstest.Item{
|
||||
ModTime: fstest.Time("2020-12-31T04:05:06.499999999Z"),
|
||||
Path: "large.txt",
|
||||
Size: -1, // use unknown size during upload
|
||||
}
|
||||
const contentSize = 2048
|
||||
contents := random.String(contentSize)
|
||||
buf := bytes.NewBufferString(contents)
|
||||
uploadHash := hash.NewMultiHasher()
|
||||
in := io.TeeReader(buf, uploadHash)
|
||||
|
||||
file.Size = -1
|
||||
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
||||
ctx := context.TODO()
|
||||
obj, err := f.Features().PutStream(ctx, in, obji)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, obj)
|
||||
remoteTarget := "large.txt (copy)"
|
||||
objTarget, err := f.Features().Copy(ctx, obj, remoteTarget)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, objTarget)
|
||||
require.Equal(t, obj.Size(), objTarget.Size())
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
@@ -59,17 +59,7 @@ func (d *Directory) candidates() []upstream.Entry {
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
entries, err := o.fs.actionEntries(o.candidates()...)
|
||||
if err == fs.ErrorPermissionDenied {
|
||||
// There are no candidates in this object which can be written to
|
||||
// So attempt to create a new object instead
|
||||
newO, err := o.fs.put(ctx, in, src, false, options...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update current object
|
||||
*o = *newO.(*Object)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(entries) == 1 {
|
||||
|
||||
@@ -17,9 +17,7 @@ func init() {
|
||||
type EpFF struct{}
|
||||
|
||||
func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath string) (*upstream.Fs, error) {
|
||||
ch := make(chan *upstream.Fs, len(upstreams))
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
ch := make(chan *upstream.Fs)
|
||||
for _, u := range upstreams {
|
||||
u := u // Closure
|
||||
go func() {
|
||||
@@ -32,10 +30,16 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath stri
|
||||
}()
|
||||
}
|
||||
var u *upstream.Fs
|
||||
for range upstreams {
|
||||
for i := 0; i < len(upstreams); i++ {
|
||||
u = <-ch
|
||||
if u != nil {
|
||||
break
|
||||
// close remaining goroutines
|
||||
go func(num int) {
|
||||
defer close(ch)
|
||||
for i := 0; i < num; i++ {
|
||||
<-ch
|
||||
}
|
||||
}(len(upstreams) - 1 - i)
|
||||
}
|
||||
}
|
||||
if u == nil {
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package union
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func (f *Fs) TestInternalReadOnly(t *testing.T) {
|
||||
if f.name != "TestUnionRO" {
|
||||
t.Skip("Only on RO union")
|
||||
}
|
||||
dir := "TestInternalReadOnly"
|
||||
ctx := context.Background()
|
||||
rofs := f.upstreams[len(f.upstreams)-1]
|
||||
assert.False(t, rofs.IsWritable())
|
||||
|
||||
// Put a file onto the read only fs
|
||||
contents := random.String(50)
|
||||
file1 := fstest.NewItem(dir+"/file.txt", contents, time.Now())
|
||||
_, obj1 := fstests.PutTestContents(ctx, t, rofs, &file1, contents, true)
|
||||
|
||||
// Check read from readonly fs via union
|
||||
o, err := f.NewObject(ctx, file1.Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(50), o.Size())
|
||||
|
||||
// Now call Update on the union Object with new data
|
||||
contents2 := random.String(100)
|
||||
file2 := fstest.NewItem(dir+"/file.txt", contents2, time.Now())
|
||||
in := bytes.NewBufferString(contents2)
|
||||
src := object.NewStaticObjectInfo(file2.Path, file2.ModTime, file2.Size, true, nil, nil)
|
||||
err = o.Update(ctx, in, src)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(100), o.Size())
|
||||
|
||||
// Check we read the new object via the union
|
||||
o, err = f.NewObject(ctx, file1.Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(100), o.Size())
|
||||
|
||||
// Remove the object
|
||||
assert.NoError(t, o.Remove(ctx))
|
||||
|
||||
// Check we read the old object in the read only layer now
|
||||
o, err = f.NewObject(ctx, file1.Path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(50), o.Size())
|
||||
|
||||
// Remove file and dir from read only fs
|
||||
assert.NoError(t, obj1.Remove(ctx))
|
||||
assert.NoError(t, rofs.Rmdir(ctx, dir))
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("ReadOnly", f.TestInternalReadOnly)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
@@ -2,15 +2,13 @@
|
||||
package union_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -26,28 +24,17 @@ func TestIntegration(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func makeTestDirs(t *testing.T, n int) (dirs []string, clean func()) {
|
||||
for i := 1; i <= n; i++ {
|
||||
dir, err := ioutil.TempDir("", fmt.Sprintf("rclone-union-test-%d", n))
|
||||
require.NoError(t, err)
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
clean = func() {
|
||||
for _, dir := range dirs {
|
||||
err := os.RemoveAll(dir)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
return dirs, clean
|
||||
}
|
||||
|
||||
func TestStandard(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-standard1")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-standard2")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-standard3")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
|
||||
name := "TestUnion"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
@@ -67,9 +54,13 @@ func TestRO(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + ":ro " + dirs[2] + ":ro"
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-ro1")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-ro2")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-ro3")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + ":ro " + tempdir3 + ":ro"
|
||||
name := "TestUnionRO"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
@@ -89,9 +80,13 @@ func TestNC(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + ":nc " + dirs[2] + ":nc"
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-nc1")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-nc2")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-nc3")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + ":nc " + tempdir3 + ":nc"
|
||||
name := "TestUnionNC"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
@@ -111,9 +106,13 @@ func TestPolicy1(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy11")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy12")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy13")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
|
||||
name := "TestUnionPolicy1"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
@@ -133,9 +132,13 @@ func TestPolicy2(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy21")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy22")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy23")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
|
||||
name := "TestUnionPolicy2"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
@@ -155,9 +158,13 @@ func TestPolicy3(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
dirs, clean := makeTestDirs(t, 3)
|
||||
defer clean()
|
||||
upstreams := dirs[0] + " " + dirs[1] + " " + dirs[2]
|
||||
tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy31")
|
||||
tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy32")
|
||||
tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy33")
|
||||
require.NoError(t, os.MkdirAll(tempdir1, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir2, 0744))
|
||||
require.NoError(t, os.MkdirAll(tempdir3, 0744))
|
||||
upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
|
||||
name := "TestUnionPolicy3"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -63,7 +62,7 @@ type Entry interface {
|
||||
// New creates a new Fs based on the
|
||||
// string formatted `type:root_path(:ro/:nc)`
|
||||
func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs, error) {
|
||||
configName, fsPath, err := fspath.SplitFs(remote)
|
||||
_, configName, fsPath, err := fs.ParseRemote(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -84,13 +83,15 @@ func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs
|
||||
f.creatable = false
|
||||
fsPath = fsPath[0 : len(fsPath)-3]
|
||||
}
|
||||
remote = configName + fsPath
|
||||
rFs, err := cache.Get(ctx, remote)
|
||||
if configName != "local" {
|
||||
fsPath = configName + ":" + fsPath
|
||||
}
|
||||
rFs, err := cache.Get(ctx, fsPath)
|
||||
if err != nil && err != fs.ErrorIsFile {
|
||||
return nil, err
|
||||
}
|
||||
f.RootFs = rFs
|
||||
rootString := path.Join(remote, filepath.ToSlash(root))
|
||||
rootString := path.Join(fsPath, filepath.ToSlash(root))
|
||||
myFs, err := cache.Get(ctx, rootString)
|
||||
if err != nil && err != fs.ErrorIsFile {
|
||||
return nil, err
|
||||
|
||||
@@ -196,10 +196,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// If we have a bearer token command and it has expired then refresh it
|
||||
if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
|
||||
fs.Debugf(f, "Bearer token expired: %v", err)
|
||||
@@ -273,7 +270,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
// does not exist
|
||||
@@ -631,7 +628,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
@@ -803,7 +800,7 @@ func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
return err == nil
|
||||
}
|
||||
@@ -825,7 +822,7 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
// Check if it already exists. The response code for this isn't
|
||||
@@ -914,7 +911,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rmdir failed")
|
||||
@@ -977,7 +974,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Copy call failed")
|
||||
@@ -1073,7 +1070,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "DirMove MOVE call failed")
|
||||
@@ -1115,7 +1112,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "about call failed")
|
||||
@@ -1243,7 +1240,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1294,7 +1291,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// Give the WebDAV server a chance to get its internal state in order after the
|
||||
@@ -1321,7 +1318,7 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
}
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -153,10 +153,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
@@ -229,7 +226,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -471,7 +468,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// fmt.Printf("CreateDir %q Error: %s\n", path, err.Error())
|
||||
@@ -546,9 +543,6 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
|
||||
var body []byte
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if err != nil {
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
@@ -591,9 +585,6 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro
|
||||
var body []byte
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if err != nil {
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
@@ -667,9 +658,6 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
|
||||
var body []byte
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if err != nil {
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
@@ -822,7 +810,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if apiErr, ok := err.(*api.ErrorResponse); ok {
|
||||
@@ -860,7 +848,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -877,7 +865,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -1011,7 +999,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -1044,7 +1032,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -1059,7 +1047,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1083,7 +1071,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -1101,7 +1089,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
return err
|
||||
|
||||
@@ -257,10 +257,7 @@ var retryErrorCodes = []int{
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
authRetry := false
|
||||
|
||||
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
|
||||
@@ -357,7 +354,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -372,7 +369,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err := configstruct.Set(m, opt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setupRegion(m)
|
||||
|
||||
root = parsePath(root)
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
@@ -454,7 +450,7 @@ OUTER:
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return found, errors.Wrap(err, "couldn't list files")
|
||||
@@ -559,7 +555,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
@@ -647,7 +643,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
|
||||
params.Set("filename", name)
|
||||
params.Set("parent_id", parent)
|
||||
params.Set("override-name-exist", strconv.FormatBool(true))
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(in, nil, "content", name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make multipart upload")
|
||||
}
|
||||
@@ -668,7 +664,7 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
|
||||
var uploadResponse *api.UploadResponse
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "upload error")
|
||||
@@ -750,7 +746,7 @@ func (f *Fs) deleteObject(ctx context.Context, id string) (err error) {
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "delete object failed")
|
||||
@@ -820,7 +816,7 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e
|
||||
var result *api.ItemInfo
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "rename failed")
|
||||
@@ -873,7 +869,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var result *api.ItemList
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, ©File, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't copy file")
|
||||
@@ -918,7 +914,7 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item,
|
||||
var result *api.ItemList
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "move failed")
|
||||
@@ -1185,7 +1181,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -47,7 +47,6 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/reveal"
|
||||
_ "github.com/rclone/rclone/cmd/rmdir"
|
||||
_ "github.com/rclone/rclone/cmd/rmdirs"
|
||||
_ "github.com/rclone/rclone/cmd/selfupdate"
|
||||
_ "github.com/rclone/rclone/cmd/serve"
|
||||
_ "github.com/rclone/rclone/cmd/settier"
|
||||
_ "github.com/rclone/rclone/cmd/sha1sum"
|
||||
|
||||
@@ -29,8 +29,8 @@ rclone config.
|
||||
|
||||
Use the --auth-no-open-browser to prevent rclone to open auth
|
||||
link in default browser automatically.`,
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 3, command, args)
|
||||
return config.Authorize(context.Background(), args, noAutoBrowser)
|
||||
config.Authorize(context.Background(), args, noAutoBrowser)
|
||||
},
|
||||
}
|
||||
|
||||
21
cmd/cmd.go
21
cmd/cmd.go
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config/configflags"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
@@ -36,7 +35,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/rc/rcflags"
|
||||
"github.com/rclone/rclone/fs/rc/rcserver"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/buildinfo"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/terminal"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -75,13 +73,9 @@ const (
|
||||
|
||||
// ShowVersion prints the version to stdout
|
||||
func ShowVersion() {
|
||||
linking, tagString := buildinfo.GetLinkingAndTags()
|
||||
fmt.Printf("rclone %s\n", fs.Version)
|
||||
fmt.Printf("- os/type: %s\n", runtime.GOOS)
|
||||
fmt.Printf("- os/arch: %s\n", runtime.GOARCH)
|
||||
fmt.Printf("- go/version: %s\n", runtime.Version())
|
||||
fmt.Printf("- go/linking: %s\n", linking)
|
||||
fmt.Printf("- go/tags: %s\n", tagString)
|
||||
fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
fmt.Printf("- go version: %s\n", runtime.Version())
|
||||
}
|
||||
|
||||
// NewFsFile creates an Fs from a name but may point to a file.
|
||||
@@ -89,7 +83,7 @@ func ShowVersion() {
|
||||
// It returns a string with the file name if points to a file
|
||||
// otherwise "".
|
||||
func NewFsFile(remote string) (fs.Fs, string) {
|
||||
_, fsPath, err := fspath.SplitFs(remote)
|
||||
_, _, fsPath, err := fs.ParseRemote(remote)
|
||||
if err != nil {
|
||||
err = fs.CountError(err)
|
||||
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
||||
@@ -388,12 +382,6 @@ func initConfig() {
|
||||
// Finish parsing any command line flags
|
||||
configflags.SetFlags(ci)
|
||||
|
||||
// Load the config
|
||||
configfile.LoadConfig(ctx)
|
||||
|
||||
// Start accounting
|
||||
accounting.Start(ctx)
|
||||
|
||||
// Hide console window
|
||||
if ci.NoConsole {
|
||||
terminal.HideConsole()
|
||||
@@ -553,9 +541,6 @@ func Main() {
|
||||
setupRootCommand(Root)
|
||||
AddBackendFlags()
|
||||
if err := Root.Execute(); err != nil {
|
||||
if strings.HasPrefix(err.Error(), "unknown command") {
|
||||
Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath())
|
||||
}
|
||||
log.Fatalf("Fatal error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package cmount
|
||||
|
||||
// ProvidedBy returns true if the rclone build for the given OS
|
||||
// provides support for lib/cgo-fuse
|
||||
func ProvidedBy(osName string) bool {
|
||||
return osName == "windows" || osName == "darwin"
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
|
||||
func init() {
|
||||
name := "cmount"
|
||||
cmountOnly := ProvidedBy(runtime.GOOS)
|
||||
cmountOnly := runtime.GOOS == "windows" || runtime.GOOS == "darwin"
|
||||
if cmountOnly {
|
||||
name = "mount"
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package copyurl
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
@@ -14,17 +13,15 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
autoFilename = false
|
||||
printFilename = false
|
||||
stdout = false
|
||||
noClobber = false
|
||||
autoFilename = false
|
||||
stdout = false
|
||||
noClobber = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path")
|
||||
flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename")
|
||||
flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name")
|
||||
flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file")
|
||||
}
|
||||
@@ -36,16 +33,15 @@ var commandDefinition = &cobra.Command{
|
||||
Download a URL's content and copy it to the destination without saving
|
||||
it in temporary storage.
|
||||
|
||||
Setting ` + "`--auto-filename`" + `will cause the file name to be retrieved from
|
||||
Setting --auto-filename will cause the file name to be retrieved from
|
||||
the from URL (after any redirections) and used in the destination
|
||||
path. With ` + "`--print-filename`" + ` in addition, the resuling file name will
|
||||
be printed.
|
||||
path.
|
||||
|
||||
Setting ` + "`--no-clobber`" + ` will prevent overwriting file on the
|
||||
Setting --no-clobber will prevent overwriting file on the
|
||||
destination if there is one with the same name.
|
||||
|
||||
Setting ` + "`--stdout`" + ` or making the output file name ` + "`-`" + `
|
||||
will cause the output to be written to standard output.
|
||||
Setting --stdout or making the output file name "-" will cause the
|
||||
output to be written to standard output.
|
||||
`,
|
||||
RunE: func(command *cobra.Command, args []string) (err error) {
|
||||
cmd.CheckArgs(1, 2, command, args)
|
||||
@@ -65,14 +61,10 @@ will cause the output to be written to standard output.
|
||||
}
|
||||
}
|
||||
cmd.Run(true, true, command, func() error {
|
||||
var dst fs.Object
|
||||
if stdout {
|
||||
err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
|
||||
} else {
|
||||
dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber)
|
||||
if printFilename && err == nil && dst != nil {
|
||||
fmt.Println(dst.Remote())
|
||||
}
|
||||
_, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber)
|
||||
}
|
||||
return err
|
||||
})
|
||||
|
||||
@@ -32,8 +32,8 @@ By default ` + "`dedupe`" + ` interactively finds files with duplicate
|
||||
names and offers to delete all but one or rename them to be
|
||||
different. This is known as deduping by name.
|
||||
|
||||
Deduping by name is only useful with a small group of backends (e.g. Google Drive,
|
||||
Opendrive) that can have duplicate file names. It can be run on wrapping backends
|
||||
Deduping by name is only useful with backends like Google Drive which
|
||||
can have duplicate file names. It can be run on wrapping backends
|
||||
(e.g. crypt) if they wrap a backend which supports duplicate file
|
||||
names.
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ When uses with the -l flag it lists the types too.
|
||||
}
|
||||
for _, remote := range remotes {
|
||||
if listLong {
|
||||
remoteType := config.FileGet(remote, "type")
|
||||
remoteType := config.FileGet(remote, "type", "UNKNOWN")
|
||||
fmt.Printf("%-*s %s\n", maxlen+1, remote+":", remoteType)
|
||||
} else {
|
||||
fmt.Printf("%s:\n", remote)
|
||||
|
||||
@@ -181,15 +181,6 @@ func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Invalidate a leaf in a directory
|
||||
func (d *Dir) invalidateEntry(dirNode fusefs.Node, leaf string) {
|
||||
fs.Debugf(dirNode, "Invalidating %q", leaf)
|
||||
err := d.fsys.server.InvalidateEntry(dirNode, leaf)
|
||||
if err != nil {
|
||||
fs.Debugf(dirNode, "Failed to invalidate %q: %v", leaf, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
var _ fusefs.NodeRenamer = (*Dir)(nil)
|
||||
|
||||
@@ -206,13 +197,6 @@ func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs
|
||||
return translateError(err)
|
||||
}
|
||||
|
||||
// Invalidate the new directory entry so it gets re-read (in
|
||||
// the background otherwise we cause a deadlock)
|
||||
//
|
||||
// See https://github.com/rclone/rclone/issues/4977 for why
|
||||
go d.invalidateEntry(newDir, req.NewName)
|
||||
//go d.invalidateEntry(d, req.OldName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -20,9 +20,8 @@ import (
|
||||
// FS represents the top level filing system
|
||||
type FS struct {
|
||||
*vfs.VFS
|
||||
f fs.Fs
|
||||
opt *mountlib.Options
|
||||
server *fusefs.Server
|
||||
f fs.Fs
|
||||
opt *mountlib.Options
|
||||
}
|
||||
|
||||
// Check interface satisfied
|
||||
|
||||
@@ -91,12 +91,12 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error
|
||||
}
|
||||
|
||||
filesys := NewFS(VFS, opt)
|
||||
filesys.server = fusefs.New(c, nil)
|
||||
server := fusefs.New(c, nil)
|
||||
|
||||
// Serve the mount point in the background returning error to errChan
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
err := filesys.server.Serve(filesys)
|
||||
err := server.Serve(filesys)
|
||||
closeErr := c.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/cmount"
|
||||
_ "github.com/rclone/rclone/cmd/mount"
|
||||
_ "github.com/rclone/rclone/cmd/mount2"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -21,7 +20,6 @@ import (
|
||||
|
||||
func TestRc(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configfile.LoadConfig(ctx)
|
||||
mount := rc.Calls.Get("mount/mount")
|
||||
assert.NotNil(t, mount)
|
||||
unmount := rc.Calls.Get("mount/unmount")
|
||||
|
||||
@@ -3,13 +3,10 @@ package rcd
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/rc/rcflags"
|
||||
"github.com/rclone/rclone/fs/rc/rcserver"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -51,22 +48,6 @@ See the [rc documentation](/rc/) for more info on the rc flags.
|
||||
log.Fatal("rc server not configured")
|
||||
}
|
||||
|
||||
// Notify stopping on exit
|
||||
var finaliseOnce sync.Once
|
||||
finalise := func() {
|
||||
finaliseOnce.Do(func() {
|
||||
_ = sysdnotify.Stopping()
|
||||
})
|
||||
}
|
||||
fnHandle := atexit.Register(finalise)
|
||||
defer atexit.Unregister(fnHandle)
|
||||
|
||||
// Notify ready to systemd
|
||||
if err := sysdnotify.Ready(); err != nil {
|
||||
log.Fatalf("failed to notify ready to systemd: %v", err)
|
||||
}
|
||||
|
||||
s.Wait()
|
||||
finalise()
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
package selfupdate
|
||||
|
||||
// Note: "|" will be replaced by backticks in the help string below
|
||||
var selfUpdateHelp string = `
|
||||
This command downloads the latest release of rclone and replaces
|
||||
the currently running binary. The download is verified with a hashsum
|
||||
and cryptographically signed signature.
|
||||
|
||||
If used without flags (or with implied |--stable| flag), this command
|
||||
will install the latest stable release. However, some issues may be fixed
|
||||
(or features added) only in the latest beta release. In such cases you should
|
||||
run the command with the |--beta| flag, i.e. |rclone selfupdate --beta|.
|
||||
You can check in advance what version would be installed by adding the
|
||||
|--check| flag, then repeat the command without it when you are satisfied.
|
||||
|
||||
Sometimes the rclone team may recommend you a concrete beta or stable
|
||||
rclone release to troubleshoot your issue or add a bleeding edge feature.
|
||||
The |--version VER| flag, if given, will update to the concrete version
|
||||
instead of the latest one. If you omit micro version from |VER| (for
|
||||
example |1.53|), the latest matching micro version will be used.
|
||||
|
||||
Upon successful update rclone will print a message that contains a previous
|
||||
version number. You will need it if you later decide to revert your update
|
||||
for some reason. Then you'll have to note the previous version and run the
|
||||
following command: |rclone selfupdate [--beta] OLDVER|.
|
||||
If the old version contains only dots and digits (for example |v1.54.0|)
|
||||
then it's a stable release so you won't need the |--beta| flag. Beta releases
|
||||
have an additional information similar to |v1.54.0-beta.5111.06f1c0c61|.
|
||||
(if you are a developer and use a locally built rclone, the version number
|
||||
will end with |-DEV|, you will have to rebuild it as it obvisously can't
|
||||
be distributed).
|
||||
|
||||
If you previously installed rclone via a package manager, the package may
|
||||
include local documentation or configure services. You may wish to update
|
||||
with the flag |--package deb| or |--package rpm| (whichever is correct for
|
||||
your OS) to update these too. This command with the default |--package zip|
|
||||
will update only the rclone executable so the local manual may become
|
||||
inaccurate after it.
|
||||
|
||||
The |rclone mount| command (https://rclone.org/commands/rclone_mount/) may
|
||||
or may not support extended FUSE options depending on the build and OS.
|
||||
|selfupdate| will refuse to update if the capability would be discarded.
|
||||
|
||||
Note: Windows forbids deletion of a currently running executable so this
|
||||
command will rename the old executable to 'rclone.old.exe' upon success.
|
||||
|
||||
Please note that this command was not available before rclone version 1.55.
|
||||
If it fails for you with the message |unknown command "selfupdate"| then
|
||||
you will need to update manually following the install instructions located
|
||||
at https://rclone.org/install/
|
||||
`
|
||||
@@ -1,480 +0,0 @@
|
||||
package selfupdate
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/cmount"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/lib/buildinfo"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
versionCmd "github.com/rclone/rclone/cmd/version"
|
||||
)
|
||||
|
||||
// Options contains options for the self-update command
|
||||
type Options struct {
|
||||
Check bool
|
||||
Output string // output path
|
||||
Beta bool // mutually exclusive with Stable (false means "stable")
|
||||
Stable bool // mutually exclusive with Beta
|
||||
Version string
|
||||
Package string // package format: zip, deb, rpm (empty string means "zip")
|
||||
}
|
||||
|
||||
// Opt is options set via command line
|
||||
var Opt = Options{}
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(cmdSelfUpdate)
|
||||
cmdFlags := cmdSelfUpdate.Flags()
|
||||
flags.BoolVarP(cmdFlags, &Opt.Check, "check", "", Opt.Check, "Check for latest release, do not download.")
|
||||
flags.StringVarP(cmdFlags, &Opt.Output, "output", "", Opt.Output, "Save the downloaded binary at a given path (default: replace running binary)")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Stable, "stable", "", Opt.Stable, "Install stable release (this is the default)")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Beta, "beta", "", Opt.Beta, "Install beta release.")
|
||||
flags.StringVarP(cmdFlags, &Opt.Version, "version", "", Opt.Version, "Install the given rclone version (default: latest)")
|
||||
flags.StringVarP(cmdFlags, &Opt.Package, "package", "", Opt.Package, "Package format: zip|deb|rpm (default: zip)")
|
||||
}
|
||||
|
||||
var cmdSelfUpdate = &cobra.Command{
|
||||
Use: "selfupdate",
|
||||
Aliases: []string{"self-update"},
|
||||
Short: `Update the rclone binary.`,
|
||||
Long: strings.ReplaceAll(selfUpdateHelp, "|", "`"),
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 0, command, args)
|
||||
if Opt.Package == "" {
|
||||
Opt.Package = "zip"
|
||||
}
|
||||
gotActionFlags := Opt.Stable || Opt.Beta || Opt.Output != "" || Opt.Version != "" || Opt.Package != "zip"
|
||||
if Opt.Check && !gotActionFlags {
|
||||
versionCmd.CheckVersion()
|
||||
return
|
||||
}
|
||||
if Opt.Package != "zip" {
|
||||
if Opt.Package != "deb" && Opt.Package != "rpm" {
|
||||
log.Fatalf("--package should be one of zip|deb|rpm")
|
||||
}
|
||||
if runtime.GOOS != "linux" {
|
||||
log.Fatalf(".deb and .rpm packages are supported only on Linux")
|
||||
} else if os.Geteuid() != 0 && !Opt.Check {
|
||||
log.Fatalf(".deb and .rpm must be installed by root")
|
||||
}
|
||||
if Opt.Output != "" && !Opt.Check {
|
||||
fmt.Println("Warning: --output is ignored with --package deb|rpm")
|
||||
}
|
||||
}
|
||||
if err := InstallUpdate(context.Background(), &Opt); err != nil {
|
||||
log.Fatalf("Error: %v", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// GetVersion can get the latest release number from the download site
|
||||
// or massage a stable release number - prepend semantic "v" prefix
|
||||
// or find the latest micro release for a given major.minor release.
|
||||
// Note: this will not be applied to beta releases.
|
||||
func GetVersion(ctx context.Context, beta bool, version string) (newVersion, siteURL string, err error) {
|
||||
siteURL = "https://downloads.rclone.org"
|
||||
if beta {
|
||||
siteURL = "https://beta.rclone.org"
|
||||
}
|
||||
|
||||
if version == "" {
|
||||
// Request the latest release number from the download site
|
||||
_, newVersion, _, err = versionCmd.GetVersion(siteURL + "/version.txt")
|
||||
return
|
||||
}
|
||||
|
||||
newVersion = version
|
||||
if version[0] != 'v' {
|
||||
newVersion = "v" + version
|
||||
}
|
||||
if beta {
|
||||
return
|
||||
}
|
||||
|
||||
if valid, _ := regexp.MatchString(`^v\d+\.\d+(\.\d+)?$`, newVersion); !valid {
|
||||
return "", siteURL, errors.New("invalid semantic version")
|
||||
}
|
||||
|
||||
// Find the latest stable micro release
|
||||
if strings.Count(newVersion, ".") == 1 {
|
||||
html, err := downloadFile(ctx, siteURL)
|
||||
if err != nil {
|
||||
return "", siteURL, errors.Wrap(err, "failed to get list of releases")
|
||||
}
|
||||
reSubver := fmt.Sprintf(`href="\./%s\.\d+/"`, regexp.QuoteMeta(newVersion))
|
||||
allSubvers := regexp.MustCompile(reSubver).FindAllString(string(html), -1)
|
||||
if allSubvers == nil {
|
||||
return "", siteURL, errors.New("could not find the minor release")
|
||||
}
|
||||
// Use the fact that releases in the index are sorted by date
|
||||
lastSubver := allSubvers[len(allSubvers)-1]
|
||||
newVersion = lastSubver[8 : len(lastSubver)-2]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// InstallUpdate performs rclone self-update
|
||||
func InstallUpdate(ctx context.Context, opt *Options) error {
|
||||
// Find the latest release number
|
||||
if opt.Stable && opt.Beta {
|
||||
return errors.New("--stable and --beta are mutually exclusive")
|
||||
}
|
||||
|
||||
gotCmount := false
|
||||
for _, tag := range buildinfo.Tags {
|
||||
if tag == "cmount" {
|
||||
gotCmount = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if gotCmount && !cmount.ProvidedBy(runtime.GOOS) {
|
||||
return errors.New("updating would discard the mount FUSE capability, aborting")
|
||||
}
|
||||
|
||||
newVersion, siteURL, err := GetVersion(ctx, opt.Beta, opt.Version)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to detect new version")
|
||||
}
|
||||
|
||||
oldVersion := fs.Version
|
||||
if newVersion == oldVersion {
|
||||
fmt.Println("rclone is up to date")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Install .deb/.rpm package if requested by user
|
||||
if opt.Package == "deb" || opt.Package == "rpm" {
|
||||
if opt.Check {
|
||||
fmt.Println("Warning: --package flag is ignored in --check mode")
|
||||
} else {
|
||||
err := installPackage(ctx, opt.Beta, newVersion, siteURL, opt.Package)
|
||||
if err == nil {
|
||||
fmt.Printf("Successfully updated rclone package from version %s to version %s\n", oldVersion, newVersion)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Get the current executable path
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to find executable")
|
||||
}
|
||||
|
||||
targetFile := opt.Output
|
||||
if targetFile == "" {
|
||||
targetFile = executable
|
||||
}
|
||||
|
||||
if opt.Check {
|
||||
fmt.Printf("Without --check this would install rclone version %s at %s\n", newVersion, targetFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Make temporary file names and check for possible access errors in advance
|
||||
var newFile string
|
||||
if newFile, err = makeRandomExeName(targetFile, "new"); err != nil {
|
||||
return err
|
||||
}
|
||||
savedFile := ""
|
||||
if runtime.GOOS == "windows" {
|
||||
savedFile = targetFile
|
||||
if strings.HasSuffix(savedFile, ".exe") {
|
||||
savedFile = savedFile[:len(savedFile)-4]
|
||||
}
|
||||
savedFile += ".old.exe"
|
||||
}
|
||||
|
||||
if savedFile == executable || newFile == executable {
|
||||
return fmt.Errorf("%s: a temporary file would overwrite the executable, specify a different --output path", targetFile)
|
||||
}
|
||||
|
||||
if err := verifyAccess(targetFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Download the update as a temporary file
|
||||
err = downloadUpdate(ctx, opt.Beta, newVersion, siteURL, newFile, "zip")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update rclone")
|
||||
}
|
||||
|
||||
err = replaceExecutable(targetFile, newFile, savedFile)
|
||||
if err == nil {
|
||||
fmt.Printf("Successfully updated rclone from version %s to version %s\n", oldVersion, newVersion)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func installPackage(ctx context.Context, beta bool, version, siteURL, packageFormat string) error {
|
||||
tempFile, err := ioutil.TempFile("", "rclone.*."+packageFormat)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to write temporary package")
|
||||
}
|
||||
packageFile := tempFile.Name()
|
||||
_ = tempFile.Close()
|
||||
defer func() {
|
||||
if rmErr := os.Remove(packageFile); rmErr != nil {
|
||||
fs.Errorf(nil, "%s: could not remove temporary package: %v", packageFile, rmErr)
|
||||
}
|
||||
}()
|
||||
if err := downloadUpdate(ctx, beta, version, siteURL, packageFile, packageFormat); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
packageCommand := "dpkg"
|
||||
if packageFormat == "rpm" {
|
||||
packageCommand = "rpm"
|
||||
}
|
||||
cmd := exec.Command(packageCommand, "-i", packageFile)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to run %s: %v", packageCommand, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceExecutable(targetFile, newFile, savedFile string) error {
|
||||
// Copy permission bits from the old executable
|
||||
// (it was extracted with mode 0755)
|
||||
fileInfo, err := os.Lstat(targetFile)
|
||||
if err == nil {
|
||||
if err = os.Chmod(newFile, fileInfo.Mode()); err != nil {
|
||||
return errors.Wrap(err, "failed to set permission")
|
||||
}
|
||||
}
|
||||
|
||||
if err = os.Remove(targetFile); os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if err != nil && savedFile != "" {
|
||||
// Windows forbids removal of a running executable so we rename it.
|
||||
// For starters, rename download as the original file with ".old.exe" appended.
|
||||
var saveErr error
|
||||
if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
|
||||
saveErr = nil
|
||||
}
|
||||
if saveErr == nil {
|
||||
saveErr = os.Rename(targetFile, savedFile)
|
||||
}
|
||||
if saveErr != nil {
|
||||
// The ".old" file cannot be removed or cannot be renamed to.
|
||||
// This usually means that the running executable has a name with ".old".
|
||||
// This can happen in very rare cases, but we ought to handle it.
|
||||
// Try inserting a randomness in the name to mitigate it.
|
||||
fs.Debugf(nil, "%s: cannot replace old file, randomizing name", savedFile)
|
||||
|
||||
savedFile, saveErr = makeRandomExeName(targetFile, "old")
|
||||
if saveErr == nil {
|
||||
if saveErr = os.Remove(savedFile); os.IsNotExist(saveErr) {
|
||||
saveErr = nil
|
||||
}
|
||||
}
|
||||
if saveErr == nil {
|
||||
saveErr = os.Rename(targetFile, savedFile)
|
||||
}
|
||||
}
|
||||
if saveErr == nil {
|
||||
fmt.Printf("The old executable was saved as %s\n", savedFile)
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = os.Rename(newFile, targetFile)
|
||||
}
|
||||
if err != nil {
|
||||
if rmErr := os.Remove(newFile); rmErr != nil {
|
||||
fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeRandomExeName(baseName, extension string) (string, error) {
|
||||
const maxAttempts = 5
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
if strings.HasSuffix(baseName, ".exe") {
|
||||
baseName = baseName[:len(baseName)-4]
|
||||
}
|
||||
extension += ".exe"
|
||||
}
|
||||
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
filename := fmt.Sprintf("%s.%s.%s", baseName, random.String(4), extension)
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
return filename, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("cannot find a file name like %s.xxxx.%s", baseName, extension)
|
||||
}
|
||||
|
||||
func downloadUpdate(ctx context.Context, beta bool, version, siteURL, newFile, packageFormat string) error {
|
||||
osName := runtime.GOOS
|
||||
arch := runtime.GOARCH
|
||||
if arch == "darwin" {
|
||||
arch = "osx"
|
||||
}
|
||||
|
||||
archiveFilename := fmt.Sprintf("rclone-%s-%s-%s.%s", version, osName, arch, packageFormat)
|
||||
archiveURL := fmt.Sprintf("%s/%s/%s", siteURL, version, archiveFilename)
|
||||
archiveBuf, err := downloadFile(ctx, archiveURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gotHash := sha256.Sum256(archiveBuf)
|
||||
strHash := hex.EncodeToString(gotHash[:])
|
||||
fs.Debugf(nil, "downloaded release archive with hashsum %s from %s", strHash, archiveURL)
|
||||
|
||||
// CI/CD does not provide hashsums for beta releases
|
||||
if !beta {
|
||||
if err := verifyHashsum(ctx, siteURL, version, archiveFilename, gotHash[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if packageFormat == "deb" || packageFormat == "rpm" {
|
||||
if err := ioutil.WriteFile(newFile, archiveBuf, 0644); err != nil {
|
||||
return errors.Wrap(err, "cannot write temporary ."+packageFormat)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
entryName := fmt.Sprintf("rclone-%s-%s-%s/rclone", version, osName, arch)
|
||||
if runtime.GOOS == "windows" {
|
||||
entryName += ".exe"
|
||||
}
|
||||
|
||||
// Extract executable to a temporary file, then replace it by an instant rename
|
||||
err = extractZipToFile(archiveBuf, entryName, newFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fs.Debugf(nil, "extracted %s to %s", entryName, newFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyAccess(file string) error {
|
||||
admin := "root"
|
||||
if runtime.GOOS == "windows" {
|
||||
admin = "Administrator"
|
||||
}
|
||||
|
||||
fileInfo, fileErr := os.Lstat(file)
|
||||
|
||||
if fileErr != nil {
|
||||
dir := filepath.Dir(file)
|
||||
dirInfo, dirErr := os.Lstat(dir)
|
||||
if dirErr != nil {
|
||||
return dirErr
|
||||
}
|
||||
if !dirInfo.Mode().IsDir() {
|
||||
return fmt.Errorf("%s: parent path is not a directory, specify a different path using --output", dir)
|
||||
}
|
||||
if !writable(dir) {
|
||||
return fmt.Errorf("%s: directory is not writable, please run self-update as %s", dir, admin)
|
||||
}
|
||||
}
|
||||
|
||||
if fileErr == nil && !fileInfo.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s: path is not a normal file, specify a different path using --output", file)
|
||||
}
|
||||
|
||||
if fileErr == nil && !writable(file) {
|
||||
return fmt.Errorf("%s: file is not writable, run self-update as %s", file, admin)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findFileHash(buf []byte, filename string) (hash []byte, err error) {
|
||||
lines := bufio.NewScanner(bytes.NewReader(buf))
|
||||
for lines.Scan() {
|
||||
tokens := strings.Split(lines.Text(), " ")
|
||||
if len(tokens) == 2 && tokens[1] == filename {
|
||||
if hash, err := hex.DecodeString(tokens[0]); err == nil {
|
||||
return hash, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%s: unable to find hash", filename)
|
||||
}
|
||||
|
||||
func extractZipToFile(buf []byte, entryName, newFile string) error {
|
||||
zipReader, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var reader io.ReadCloser
|
||||
for _, entry := range zipReader.File {
|
||||
if entry.Name == entryName {
|
||||
reader, err = entry.Open()
|
||||
break
|
||||
}
|
||||
}
|
||||
if reader == nil || err != nil {
|
||||
return fmt.Errorf("%s: file not found in archive", entryName)
|
||||
}
|
||||
defer func() {
|
||||
_ = reader.Close()
|
||||
}()
|
||||
|
||||
err = os.Remove(newFile)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("%s: unable to create new file: %v", newFile, err)
|
||||
}
|
||||
writer, err := os.OpenFile(newFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, os.FileMode(0755))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, reader)
|
||||
_ = writer.Close()
|
||||
if err != nil {
|
||||
if rmErr := os.Remove(newFile); rmErr != nil {
|
||||
fs.Errorf(nil, "%s: could not remove temporary file: %v", newFile, rmErr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func downloadFile(ctx context.Context, url string) ([]byte, error) {
|
||||
resp, err := fshttp.NewClient(ctx).Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fs.CheckClose(resp.Body, &err)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed with %s downloading %s", resp.Status, url)
|
||||
}
|
||||
return ioutil.ReadAll(resp.Body)
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
package selfupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/testy"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetVersion(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// a beta version can only have "v" prepended
|
||||
resultVer, _, err := GetVersion(ctx, true, "1.2.3.4")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "v1.2.3.4", resultVer)
|
||||
|
||||
// but a stable version syntax should be checked
|
||||
_, _, err = GetVersion(ctx, false, "1")
|
||||
assert.Error(t, err)
|
||||
_, _, err = GetVersion(ctx, false, "1.")
|
||||
assert.Error(t, err)
|
||||
_, _, err = GetVersion(ctx, false, "1.2.")
|
||||
assert.Error(t, err)
|
||||
_, _, err = GetVersion(ctx, false, "1.2.3.4")
|
||||
assert.Error(t, err)
|
||||
|
||||
// incomplete stable version should have micro release added
|
||||
resultVer, _, err = GetVersion(ctx, false, "1.52")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "v1.52.3", resultVer)
|
||||
}
|
||||
|
||||
func makeTestDir() (testDir string, err error) {
|
||||
const maxAttempts = 5
|
||||
testDirBase := filepath.Join(os.TempDir(), "rclone-test-selfupdate.")
|
||||
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
testDir = testDirBase + random.String(4)
|
||||
err = os.MkdirAll(testDir, os.ModePerm)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestInstallOnLinux(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("this is a Linux only test")
|
||||
}
|
||||
|
||||
// Prepare for test
|
||||
ctx := context.Background()
|
||||
testDir, err := makeTestDir()
|
||||
assert.NoError(t, err)
|
||||
path := filepath.Join(testDir, "rclone")
|
||||
defer func() {
|
||||
_ = os.Chmod(path, 0644)
|
||||
_ = os.RemoveAll(testDir)
|
||||
}()
|
||||
|
||||
regexVer := regexp.MustCompile(`v[0-9]\S+`)
|
||||
|
||||
betaVer, _, err := GetVersion(ctx, true, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Must do nothing if version isn't changing
|
||||
assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path, Version: fs.Version}))
|
||||
|
||||
// Must fail on non-writable file
|
||||
assert.NoError(t, ioutil.WriteFile(path, []byte("test"), 0644))
|
||||
assert.NoError(t, os.Chmod(path, 0000))
|
||||
err = (InstallUpdate(ctx, &Options{Beta: true, Output: path}))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "run self-update as root")
|
||||
|
||||
// Must keep non-standard permissions
|
||||
assert.NoError(t, os.Chmod(path, 0644))
|
||||
assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
|
||||
|
||||
info, err := os.Stat(path)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0644), info.Mode().Perm())
|
||||
|
||||
// Must remove temporary files
|
||||
files, err := ioutil.ReadDir(testDir)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(files))
|
||||
|
||||
// Must contain valid executable
|
||||
assert.NoError(t, os.Chmod(path, 0755))
|
||||
cmd := exec.Command(path, "version")
|
||||
output, err := cmd.CombinedOutput()
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, cmd.ProcessState.Success())
|
||||
assert.Equal(t, betaVer, regexVer.FindString(string(output)))
|
||||
}
|
||||
|
||||
func TestRenameOnWindows(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("this is a Windows only test")
|
||||
}
|
||||
|
||||
// Prepare for test
|
||||
ctx := context.Background()
|
||||
|
||||
testDir, err := makeTestDir()
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(testDir)
|
||||
}()
|
||||
|
||||
path := filepath.Join(testDir, "rclone.exe")
|
||||
regexVer := regexp.MustCompile(`v[0-9]\S+`)
|
||||
|
||||
stableVer, _, err := GetVersion(ctx, false, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
betaVer, _, err := GetVersion(ctx, true, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Must not create temporary files when target doesn't exist
|
||||
assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
|
||||
|
||||
files, err := ioutil.ReadDir(testDir)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(files))
|
||||
|
||||
// Must save running executable as the "old" file
|
||||
cmdWait := exec.Command(path, "config")
|
||||
stdinWait, err := cmdWait.StdinPipe() // Make it run waiting for input
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, cmdWait.Start())
|
||||
|
||||
assert.NoError(t, InstallUpdate(ctx, &Options{Beta: false, Output: path}))
|
||||
files, err = ioutil.ReadDir(testDir)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, len(files))
|
||||
|
||||
pathOld := filepath.Join(testDir, "rclone.old.exe")
|
||||
_, err = os.Stat(pathOld)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cmd := exec.Command(path, "version")
|
||||
output, err := cmd.CombinedOutput()
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, cmd.ProcessState.Success())
|
||||
assert.Equal(t, stableVer, regexVer.FindString(string(output)))
|
||||
|
||||
cmdOld := exec.Command(pathOld, "version")
|
||||
output, err = cmdOld.CombinedOutput()
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, cmdOld.ProcessState.Success())
|
||||
assert.Equal(t, betaVer, regexVer.FindString(string(output)))
|
||||
|
||||
// Stop previous waiting executable, run new and saved executables
|
||||
_ = stdinWait.Close()
|
||||
_ = cmdWait.Wait()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
cmdWait = exec.Command(path, "config")
|
||||
stdinWait, err = cmdWait.StdinPipe()
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, cmdWait.Start())
|
||||
|
||||
cmdWaitOld := exec.Command(pathOld, "config")
|
||||
stdinWaitOld, err := cmdWaitOld.StdinPipe()
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, cmdWaitOld.Start())
|
||||
|
||||
// Updating when the "old" executable is running must produce a random "old" file
|
||||
assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
|
||||
files, err = ioutil.ReadDir(testDir)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(files))
|
||||
|
||||
// Stop all waiting executables
|
||||
_ = stdinWait.Close()
|
||||
_ = cmdWait.Wait()
|
||||
_ = stdinWaitOld.Close()
|
||||
_ = cmdWaitOld.Wait()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package selfupdate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/crypto/openpgp"
|
||||
"golang.org/x/crypto/openpgp/clearsign"
|
||||
)
|
||||
|
||||
var ncwPublicKeyPGP = `-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQGiBDuy3V0RBADVQOAF5aFiCxD3t2h6iAF2WMiaMlgZ6kX2i/u7addNkzX71VU9
|
||||
7NpI0SnsP5YWt+gEedST6OmFbtLfZWCR4KWn5XnNdjCMNhxaH6WccVqNm4ALPIqT
|
||||
59uVjkgf8RISmmoNJ1d+2wMWjQTUfwOEmoIgH6n+2MYNUKuctBrwAACflwCg1I1Q
|
||||
O/prv/5hczdpQCs+fL87DxsD/Rt7pIXvsIOZyQWbIhSvNpGalJuMkW5Jx92UjsE9
|
||||
1Ipo3Xr6SGRPgW9+NxAZAsiZfCX/19knAyNrN9blwL0rcPDnkhdGwK69kfjF+wq+
|
||||
QbogRGodbKhqY4v+cMNkKiemBuTQiWPkpKjifwNsD1fNjNKfDP3pJ64Yz7a4fuzV
|
||||
X1YwBACpKVuEen34lmcX6ziY4jq8rKibKBs4JjQCRO24kYoHDULVe+RS9krQWY5b
|
||||
e0foDhru4dsKccefK099G+WEzKVCKxupstWkTT/iJwajR8mIqd4AhD0wO9W3MCfV
|
||||
Ov8ykMDZ7qBWk1DHc87Ep3W1o8t8wq74ifV+HjhhWg8QAylXg7QlTmljayBDcmFp
|
||||
Zy1Xb29kIDxuaWNrQGNyYWlnLXdvb2QuY29tPohxBBMRCAAxBQsHCgMEAxUDAgMW
|
||||
AgECF4AWIQT79zfs6firGGBL0qyTk14C/ztU+gUCXjg2UgIZAQAKCRCTk14C/ztU
|
||||
+lmmAJ4jH5FyULzStjisuTvHLTVz6G44eQCfaR5QGZFPseenE5ic2WeQcBcmtoG5
|
||||
Ag0EO7LdgRAIAI6QdFBg3/xa1gFKPYy1ihV9eSdGqwWZGJvokWsfCvHy5180tj/v
|
||||
UNOLAJrdqglMSvevNTXe8bT65D6423AAsLhch9wq/aNqrHolTYABzxRigjcS1//T
|
||||
yln5naGUzlVQXDVfrDk3Md/NrkdOFj7r/YyMF0+iWwpFz2qAjL95i5wfVZ1kWGrT
|
||||
2AmivE1wD1sWT/Ja3FDI0NRkU0Nbz/a0TKe4ml8iLVtZXpTRbxxCCPdkHXXgSyu1
|
||||
eZ4NrF/wTJuvwGn12TJ1EF95aVkHxAUw0+KmLGdcyBG+IKuHamrsjWIAXGXV///K
|
||||
AxPgUthccQ03HMjltFsrdmen5Q034YM3eOsAAwUH/jAKiIAA8LpZmZPnt9GZ4+Ol
|
||||
Zp22VAfyfDOFl4Ol+cWjkLAgjAFsm5gnOKcRSE/9XPxnQqkhw7+ZygYuUMgTDJ99
|
||||
/5IM1UQL3ooS+oFrDaE99S8bLeOe17skcdXcA/K83VqD9m93rQRnbtD+75zqKkZn
|
||||
9WNFyKCXg5P6PFPdNYRtlQKOcwFR9mHRLUmapQSAM8Y2pCgALZ7GViKQca8/TT1T
|
||||
gZk9fJMZYGez+IlOPxTJxjn80+vywk4/wdIWSiQj+8u5RzT9sjmm77wbMVNGRqYd
|
||||
W/EemW9Zz9vi0CIvJGgbPMqcuxw8e/5lnuQ6Mi3uDR0P2RNIAhFrdZpVSME8xQaI
|
||||
RgQYEQIABgUCO7LdgQAKCRCTk14C/ztU+mLBAKC2cdFy7eLaQAvyzcE2VK6HVIjn
|
||||
JACguA00bxLQuJ4+RCJrLFZP8ZlN2sc=
|
||||
=TtR5
|
||||
-----END PGP PUBLIC KEY BLOCK-----`
|
||||
|
||||
func verifyHashsum(ctx context.Context, siteURL, version, archive string, hash []byte) error {
|
||||
sumsURL := fmt.Sprintf("%s/%s/SHA256SUMS", siteURL, version)
|
||||
sumsBuf, err := downloadFile(ctx, sumsURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fs.Debugf(nil, "downloaded hashsum list: %s", sumsURL)
|
||||
|
||||
keyRing, err := openpgp.ReadArmoredKeyRing(strings.NewReader(ncwPublicKeyPGP))
|
||||
if err != nil {
|
||||
return errors.New("unsupported signing key")
|
||||
}
|
||||
block, rest := clearsign.Decode(sumsBuf)
|
||||
// block.Bytes = block.Bytes[1:] // uncomment to test invalid signature
|
||||
_, err = openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body)
|
||||
if err != nil || len(rest) > 0 {
|
||||
return errors.New("invalid hashsum signature")
|
||||
}
|
||||
|
||||
wantHash, err := findFileHash(sumsBuf, archive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(hash, wantHash) {
|
||||
return fmt.Errorf("archive hash mismatch: want %02x vs got %02x", wantHash, hash)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
// +build !windows,!plan9,!js
|
||||
|
||||
package selfupdate
|
||||
|
||||
import (
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func writable(path string) bool {
|
||||
return unix.Access(path, unix.W_OK) == nil
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
// +build plan9 js
|
||||
|
||||
package selfupdate
|
||||
|
||||
func writable(path string) bool {
|
||||
return true
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
// +build windows
|
||||
|
||||
package selfupdate
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func writable(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
const UserWritableBit = 128
|
||||
if err == nil {
|
||||
return info.Mode().Perm()&UserWritableBit != 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -13,12 +13,12 @@ import (
|
||||
|
||||
"github.com/anacrolix/dms/soap"
|
||||
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/cmd/serve/dlna/dlnaflags"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -41,7 +41,7 @@ func startServer(t *testing.T, f fs.Fs) {
|
||||
}
|
||||
|
||||
func TestInit(t *testing.T) {
|
||||
configfile.LoadConfig(context.Background())
|
||||
config.LoadConfig(context.Background())
|
||||
|
||||
f, err := fs.NewFs(context.Background(), "testdata/files")
|
||||
l, _ := f.List(context.Background(), "")
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/cmd/serve/httplib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -61,7 +61,7 @@ var (
|
||||
func TestInit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// Configure the remote
|
||||
configfile.LoadConfig(context.Background())
|
||||
config.LoadConfig(context.Background())
|
||||
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||
// fs.Config.DumpHeaders = true
|
||||
// fs.Config.DumpBodies = true
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -65,8 +63,6 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
|
||||
|
||||
// TestResticHandler runs tests on the restic handler code, especially in append-only mode.
|
||||
func TestResticHandler(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configfile.LoadConfig(ctx)
|
||||
buf := make([]byte, 32)
|
||||
_, err := io.ReadFull(rand.Reader, buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -27,8 +27,7 @@ var commandDefinition = &cobra.Command{
|
||||
Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer unchanged files, testing by size and
|
||||
modification time or MD5SUM. Destination is updated to match
|
||||
source, including deleting files if necessary (except duplicate
|
||||
objects, see below).
|
||||
source, including deleting files if necessary.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
@@ -36,8 +35,7 @@ objects, see below).
|
||||
rclone sync -i SOURCE remote:DESTINATION
|
||||
|
||||
Note that files in the destination won't be deleted if there were any
|
||||
errors at any point. Duplicate objects (files with the same name, on
|
||||
those providers that support it) are also not yet handled.
|
||||
errors at any point.
|
||||
|
||||
It is always the contents of the directory that is synced, not the
|
||||
directory so when source:path is a directory, it's the contents of
|
||||
@@ -48,9 +46,6 @@ If dest:path doesn't exist, it is created and the source:path contents
|
||||
go there.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
|
||||
@@ -29,21 +29,14 @@ var commandDefinition = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: `Show the version number.`,
|
||||
Long: `
|
||||
Show the rclone version number, the go version, the build target OS and
|
||||
architecture, build tags and the type of executable (static or dynamic).
|
||||
Show the version number, the go version and the architecture.
|
||||
|
||||
For example:
|
||||
Eg
|
||||
|
||||
$ rclone version
|
||||
rclone v1.54
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.16
|
||||
- go/linking: static
|
||||
- go/tags: none
|
||||
|
||||
Note: before rclone version 1.55 the os/type and os/arch lines were merged,
|
||||
and the "go/version" line was tagged as "go version".
|
||||
rclone v1.41
|
||||
- os/arch: linux/amd64
|
||||
- go version: go1.10
|
||||
|
||||
If you supply the --check flag, then it will do an online check to
|
||||
compare your version with the latest release and the latest beta.
|
||||
@@ -66,7 +59,7 @@ Or
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 0, command, args)
|
||||
if check {
|
||||
CheckVersion()
|
||||
checkVersion()
|
||||
} else {
|
||||
cmd.ShowVersion()
|
||||
}
|
||||
@@ -81,8 +74,8 @@ func stripV(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// GetVersion gets the version available for download
|
||||
func GetVersion(url string) (v *semver.Version, vs string, date time.Time, err error) {
|
||||
// getVersion gets the version by checking the download repository passed in
|
||||
func getVersion(url string) (v *semver.Version, vs string, date time.Time, err error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return v, vs, date, err
|
||||
@@ -96,7 +89,9 @@ func GetVersion(url string) (v *semver.Version, vs string, date time.Time, err e
|
||||
return v, vs, date, err
|
||||
}
|
||||
vs = strings.TrimSpace(string(bodyBytes))
|
||||
vs = strings.TrimPrefix(vs, "rclone ")
|
||||
if strings.HasPrefix(vs, "rclone ") {
|
||||
vs = vs[7:]
|
||||
}
|
||||
vs = strings.TrimRight(vs, "β")
|
||||
date, err = http.ParseTime(resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
@@ -106,8 +101,9 @@ func GetVersion(url string) (v *semver.Version, vs string, date time.Time, err e
|
||||
return v, vs, date, err
|
||||
}
|
||||
|
||||
// CheckVersion checks the installed version against available downloads
|
||||
func CheckVersion() {
|
||||
// check the current version against available versions
|
||||
func checkVersion() {
|
||||
// Get Current version
|
||||
vCurrent, err := semver.NewVersion(stripV(fs.Version))
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to parse version: %v", err)
|
||||
@@ -115,7 +111,7 @@ func CheckVersion() {
|
||||
const timeFormat = "2006-01-02"
|
||||
|
||||
printVersion := func(what, url string) {
|
||||
v, vs, t, err := GetVersion(url + "version.txt")
|
||||
v, vs, t, err := getVersion(url + "version.txt")
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to get rclone %s version: %v", what, err)
|
||||
return
|
||||
|
||||
@@ -473,8 +473,3 @@ put them back in again.` >}}
|
||||
* tYYGH <tYYGH@users.noreply.github.com>
|
||||
* georne <77802995+georne@users.noreply.github.com>
|
||||
* Maxwell Calman <mcalman@MacBook-Pro.local>
|
||||
* Naveen Honest Raj <naveendurai19@gmail.com>
|
||||
* Lucas Messenger <lmesseng@cisco.com>
|
||||
* Manish Kumar <krmanish260@gmail.com>
|
||||
* x0b <x0bdev@gmail.com>
|
||||
* CERN through the CS3MESH4EOSC Project
|
||||
|
||||
@@ -392,22 +392,6 @@ See: the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
- Type: MultiEncoder
|
||||
- Default: Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8
|
||||
|
||||
#### --azureblob-public-access
|
||||
|
||||
Public access level of a container: blob, container.
|
||||
|
||||
- Config: public_access
|
||||
- Env Var: RCLONE_AZUREBLOB_PUBLIC_ACCESS
|
||||
- Type: string
|
||||
- Default: ""
|
||||
- Examples:
|
||||
- ""
|
||||
- The container and its blobs can be accessed only with an authorized request. It's a default value
|
||||
- "blob"
|
||||
- Blob data within this container can be read via anonymous request.
|
||||
- "container"
|
||||
- Allow full public read access for container and blob data.
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
### Limitations ###
|
||||
|
||||
|
||||
@@ -5,160 +5,6 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.55.0 - 2021-03-31
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.54.0...v1.55.0)
|
||||
|
||||
* New commands
|
||||
* [selfupdate](/commands/rclone_selfupdate/) (Ivan Andreev)
|
||||
* Allows rclone to update itself in-place or via a package (using `--package` flag)
|
||||
* Reads cryptographically signed signatures for non beta releases
|
||||
* Works on all OSes.
|
||||
* [test](/commands/rclone_test/) - these are test commands - use with care!
|
||||
* `histogram` - Makes a histogram of file name characters.
|
||||
* `info` - Discovers file name or other limitations for paths.
|
||||
* `makefiles` - Make a random file hierarchy for testing.
|
||||
* `memory` - Load all the objects at remote:path into memory and report memory stats.
|
||||
* New Features
|
||||
* [Connection strings](/docs/#connection-strings)
|
||||
* Config parameters can now be passed as part of the remote name as a connection string.
|
||||
* For example to do the equivalent of `--drive-shared-with-me` use `drive,shared_with_me:`
|
||||
* Make sure we don't save on the fly remote config to the config file (Nick Craig-Wood)
|
||||
* Make sure backends with additional config have a different name for caching (Nick Craig-Wood)
|
||||
* This work was sponsored by CERN, through the [CS3MESH4EOSC Project](https://cs3mesh4eosc.eu/).
|
||||
* CS3MESH4EOSC has received funding from the European Union’s Horizon 2020
|
||||
* research and innovation programme under Grant Agreement no. 863353.
|
||||
* build
|
||||
* Update go build version to go1.16 and raise minimum go version to go1.13 (Nick Craig-Wood)
|
||||
* Make a macOS ARM64 build to support Apple Silicon (Nick Craig-Wood)
|
||||
* Install macfuse 4.x instead of osxfuse 3.x (Nick Craig-Wood)
|
||||
* Use `GO386=softfloat` instead of deprecated `GO386=387` for 386 builds (Nick Craig-Wood)
|
||||
* Disable IOS builds for the time being (Nick Craig-Wood)
|
||||
* Androids builds made with up to date NDK (x0b)
|
||||
* Add an rclone user to the Docker image but don't use it by default (cynthia kwok)
|
||||
* dedupe: Make largest directory primary to minimize data moved (Saksham Khanna)
|
||||
* config
|
||||
* Wrap config library in an interface (Fionera)
|
||||
* Make config file system pluggable (Nick Craig-Wood)
|
||||
* `--config ""` or `"/notfound"` for in memory config only (Nick Craig-Wood)
|
||||
* Clear fs cache of stale entries when altering config (Nick Craig-Wood)
|
||||
* copyurl: Add option to print resulting auto-filename (albertony)
|
||||
* delete: Make `--rmdirs` obey the filters (Nick Craig-Wood)
|
||||
* docs - many fixes and reworks from edwardxml, albertony, pvalls, Ivan Andreev, Evan Harris, buengese, Alexey Tabakman
|
||||
* encoder/filename - add SCSU as tables (Klaus Post)
|
||||
* Add multiple paths support to `--compare-dest` and `--copy-dest` flag (K265)
|
||||
* filter: Make `--exclude "dir/"` equivalent to `--exclude "dir/**"` (Nick Craig-Wood)
|
||||
* fshttp: Add DSCP support with `--dscp` for QoS with differentiated services (Max Sum)
|
||||
* lib/cache: Add Delete and DeletePrefix methods (Nick Craig-Wood)
|
||||
* lib/file
|
||||
* Make pre-allocate detect disk full errors and return them (Nick Craig-Wood)
|
||||
* Don't run preallocate concurrently (Nick Craig-Wood)
|
||||
* Retry preallocate on EINTR (Nick Craig-Wood)
|
||||
* operations: Made copy and sync operations obey a RetryAfterError (Ankur Gupta)
|
||||
* rc
|
||||
* Add string alternatives for setting options over the rc (Nick Craig-Wood)
|
||||
* Add `options/local` to see the options configured in the context (Nick Craig-Wood)
|
||||
* Add `_config` parameter to set global config for just this rc call (Nick Craig-Wood)
|
||||
* Implement passing filter config with `_filter` parameter (Nick Craig-Wood)
|
||||
* Add `fscache/clear` and `fscache/entries` to control the fs cache (Nick Craig-Wood)
|
||||
* Avoid +Inf value for speed in `core/stats` (albertony)
|
||||
* Add a full set of stats to `core/stats` (Nick Craig-Wood)
|
||||
* Allow `fs=` params to be a JSON blob (Nick Craig-Wood)
|
||||
* rcd: Added systemd notification during the `rclone rcd` command. (Naveen Honest Raj)
|
||||
* rmdirs: Make `--rmdirs` obey the filters (Nick Craig-Wood)
|
||||
* version: Show build tags and type of executable (Ivan Andreev)
|
||||
* Bug Fixes
|
||||
* install.sh: make it fail on download errors (Ivan Andreev)
|
||||
* Fix excessive retries missing `--max-duration` timeout (Nick Craig-Wood)
|
||||
* Fix crash when `--low-level-retries=0` (Nick Craig-Wood)
|
||||
* Fix failed token refresh on mounts created via the rc (Nick Craig-Wood)
|
||||
* fshttp: Fix bandwidth limiting after bad merge (Nick Craig-Wood)
|
||||
* lib/atexit
|
||||
* Unregister interrupt handler once it has fired so users can interrupt again (Nick Craig-Wood)
|
||||
* Fix occasional failure to unmount with CTRL-C (Nick Craig-Wood)
|
||||
* Fix deadlock calling Finalise while Run is running (Nick Craig-Wood)
|
||||
* lib/rest: Fix multipart uploads not stopping on context cancel (Nick Craig-Wood)
|
||||
* Mount
|
||||
* Allow mounting to root directory on windows (albertony)
|
||||
* Improved handling of relative paths on windows (albertony)
|
||||
* Fix unicode issues with accented characters on macOS (Nick Craig-Wood)
|
||||
* Docs: document the new FileSecurity option in WinFsp 2021 (albertony)
|
||||
* Docs: add note about volume path syntax on windows (albertony)
|
||||
* Fix caching of old directories after renaming them (Nick Craig-Wood)
|
||||
* Update cgofuse to the latest version to bring in macfuse 4 fix (Nick Craig-Wood)
|
||||
* VFS
|
||||
* `--vfs-used-is-size` to report used space using recursive scan (tYYGH)
|
||||
* Don't set modification time if it was already correct (Nick Craig-Wood)
|
||||
* Fix Create causing windows explorer to truncate files on CTRL-C CTRL-V (Nick Craig-Wood)
|
||||
* Fix modtimes not updating when writing via cache (Nick Craig-Wood)
|
||||
* Fix modtimes changing by fractional seconds after upload (Nick Craig-Wood)
|
||||
* Fix modtime set if `--vfs-cache-mode writes`/`full` and no write (Nick Craig-Wood)
|
||||
* Rename files in cache and cancel uploads on directory rename (Nick Craig-Wood)
|
||||
* Fix directory renaming by renaming dirs cached in memory (Nick Craig-Wood)
|
||||
* Local
|
||||
* Add flag `--local-no-preallocate` (David Sze)
|
||||
* Make `nounc` an advanced option except on Windows (albertony)
|
||||
* Don't ignore preallocate disk full errors (Nick Craig-Wood)
|
||||
* Cache
|
||||
* Add `--fs-cache-expire-duration` to control the fs cache (Nick Craig-Wood)
|
||||
* Crypt
|
||||
* Add option to not encrypt data (Vesnyx)
|
||||
* Log hash ok on upload (albertony)
|
||||
* Azure Blob
|
||||
* Add container public access level support. (Manish Kumar)
|
||||
* B2
|
||||
* Fix HTML files downloaded via cloudflare (Nick Craig-Wood)
|
||||
* Box
|
||||
* Fix transfers getting stuck on token expiry after API change (Nick Craig-Wood)
|
||||
* Chunker
|
||||
* Partially implement no-rename transactions (Maxwell Calman)
|
||||
* Drive
|
||||
* Don't stop server side copy if couldn't read description (Nick Craig-Wood)
|
||||
* Pass context on to drive SDK - to help with cancellation (Nick Craig-Wood)
|
||||
* Dropbox
|
||||
* Add polling for changes support (Robert Thomas)
|
||||
* Make `--timeout 0` work properly (Nick Craig-Wood)
|
||||
* Raise priority of rate limited message to INFO to make it more noticeable (Nick Craig-Wood)
|
||||
* Fichier
|
||||
* Implement copy & move (buengese)
|
||||
* Implement public link (buengese)
|
||||
* FTP
|
||||
* Implement Shutdown method (Nick Craig-Wood)
|
||||
* Close idle connections after `--ftp-idle-timeout` (1m by default) (Nick Craig-Wood)
|
||||
* Make `--timeout 0` work properly (Nick Craig-Wood)
|
||||
* Add `--ftp-close-timeout` flag for use with awkward ftp servers (Nick Craig-Wood)
|
||||
* Retry connections and logins on 421 errors (Nick Craig-Wood)
|
||||
* Hdfs
|
||||
* Fix permissions for when directory is created (Lucas Messenger)
|
||||
* Onedrive
|
||||
* Make `--timeout 0` work properly (Nick Craig-Wood)
|
||||
* S3
|
||||
* Fix `--s3-profile` which wasn't working (Nick Craig-Wood)
|
||||
* SFTP
|
||||
* Close idle connections after `--sftp-idle-timeout` (1m by default) (Nick Craig-Wood)
|
||||
* Fix "file not found" errors for read once servers (Nick Craig-Wood)
|
||||
* Fix SetModTime stat failed: object not found with `--sftp-set-modtime=false` (Nick Craig-Wood)
|
||||
* Swift
|
||||
* Update github.com/ncw/swift to v2.0.0 (Nick Craig-Wood)
|
||||
* Implement copying large objects (nguyenhuuluan434)
|
||||
* Union
|
||||
* Fix crash when using epff policy (Nick Craig-Wood)
|
||||
* Fix union attempting to update files on a read only file system (Nick Craig-Wood)
|
||||
* Refactor to use fspath.SplitFs instead of fs.ParseRemote (Nick Craig-Wood)
|
||||
* Fix initialisation broken in refactor (Nick Craig-Wood)
|
||||
* WebDAV
|
||||
* Add support for sharepoint with NTLM authentication (Rauno Ots)
|
||||
* Make sharepoint-ntlm docs more consistent (Alex Chen)
|
||||
* Improve terminology in sharepoint-ntlm docs (Ivan Andreev)
|
||||
* Disable HTTP/2 for NTLM authentication (georne)
|
||||
* Fix sharepoint-ntlm error 401 for parallel actions (Ivan Andreev)
|
||||
* Check that purged directory really exists (Ivan Andreev)
|
||||
* Yandex
|
||||
* Make `--timeout 0` work properly (Nick Craig-Wood)
|
||||
* Zoho
|
||||
* Replace client id - you will need to `rclone config reconnect` after this (buengese)
|
||||
* Add forgotten setupRegion() to NewFs - this finally fixes regions other than EU (buengese)
|
||||
|
||||
## v1.54.1 - 2021-03-08
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.54.0...v1.54.1)
|
||||
|
||||
@@ -416,27 +416,4 @@ Choose how chunker should handle files with missing or invalid chunks.
|
||||
- "false"
|
||||
- Warn user, skip incomplete file and proceed.
|
||||
|
||||
#### --chunker-transactions
|
||||
|
||||
Choose how chunker should handle temporary files during transactions.
|
||||
|
||||
- Config: transactions
|
||||
- Env Var: RCLONE_CHUNKER_TRANSACTIONS
|
||||
- Type: string
|
||||
- Default: "rename"
|
||||
- Examples:
|
||||
- "rename"
|
||||
- Rename temporary files after a successful transaction.
|
||||
- "norename"
|
||||
- Leave temporary file names and write transaction ID to metadata file.
|
||||
- Metadata is required for no rename transactions (meta format cannot be "none").
|
||||
- If you are using norename transactions you should be careful not to downgrade Rclone
|
||||
- as older versions of Rclone don't support this transaction style and will misinterpret
|
||||
- files manipulated by norename transactions.
|
||||
- This method is EXPERIMENTAL, don't use on production systems.
|
||||
- "auto"
|
||||
- Rename or norename will be used depending on capabilities of the backend.
|
||||
- If meta format is set to "none", rename transactions will always be used.
|
||||
- This method is EXPERIMENTAL, don't use on production systems.
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
@@ -72,13 +72,11 @@ See the [global flags page](/flags/) for global options not listed here.
|
||||
* [rclone rcd](/commands/rclone_rcd/) - Run rclone listening to remote control commands only.
|
||||
* [rclone rmdir](/commands/rclone_rmdir/) - Remove the empty directory at path.
|
||||
* [rclone rmdirs](/commands/rclone_rmdirs/) - Remove empty directories under the path.
|
||||
* [rclone selfupdate](/commands/rclone_selfupdate/) - Update the rclone binary.
|
||||
* [rclone serve](/commands/rclone_serve/) - Serve a remote over a protocol.
|
||||
* [rclone settier](/commands/rclone_settier/) - Changes storage class/tier of objects in remote.
|
||||
* [rclone sha1sum](/commands/rclone_sha1sum/) - Produces an sha1sum file for all the objects in the path.
|
||||
* [rclone size](/commands/rclone_size/) - Prints the total size and number of objects in remote:path.
|
||||
* [rclone sync](/commands/rclone_sync/) - Make source and dest identical, modifying destination only.
|
||||
* [rclone test](/commands/rclone_test/) - Run a test command
|
||||
* [rclone touch](/commands/rclone_touch/) - Create new file or change file modification time.
|
||||
* [rclone tree](/commands/rclone_tree/) - List the contents of the remote in a tree like fashion.
|
||||
* [rclone version](/commands/rclone_version/) - Show the version number.
|
||||
|
||||
@@ -15,16 +15,15 @@ Copy url content to dest.
|
||||
Download a URL's content and copy it to the destination without saving
|
||||
it in temporary storage.
|
||||
|
||||
Setting `--auto-filename`will cause the file name to be retrieved from
|
||||
Setting --auto-filename will cause the file name to be retrieved from
|
||||
the from URL (after any redirections) and used in the destination
|
||||
path. With `--print-filename` in addition, the resuling file name will
|
||||
be printed.
|
||||
path.
|
||||
|
||||
Setting `--no-clobber` will prevent overwriting file on the
|
||||
Setting --no-clobber will prevent overwriting file on the
|
||||
destination if there is one with the same name.
|
||||
|
||||
Setting `--stdout` or making the output file name `-`
|
||||
will cause the output to be written to standard output.
|
||||
Setting --stdout or making the output file name "-" will cause the
|
||||
output to be written to standard output.
|
||||
|
||||
|
||||
```
|
||||
@@ -34,11 +33,10 @@ rclone copyurl https://example.com dest:path [flags]
|
||||
## Options
|
||||
|
||||
```
|
||||
-a, --auto-filename Get the file name from the URL and use it for destination file path
|
||||
-h, --help help for copyurl
|
||||
--no-clobber Prevent overwriting file with same name
|
||||
-p, --print-filename Print the resulting name from --auto-filename
|
||||
--stdout Write the output to stdout rather than a file
|
||||
-a, --auto-filename Get the file name from the URL and use it for destination file path
|
||||
-h, --help help for copyurl
|
||||
--no-clobber Prevent overwriting file with same name
|
||||
--stdout Write the output to stdout rather than a file
|
||||
```
|
||||
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
@@ -17,8 +17,8 @@ By default `dedupe` interactively finds files with duplicate
|
||||
names and offers to delete all but one or rename them to be
|
||||
different. This is known as deduping by name.
|
||||
|
||||
Deduping by name is only useful with a small group of backends (e.g. Google Drive,
|
||||
Opendrive) that can have duplicate file names. It can be run on wrapping backends
|
||||
Deduping by name is only useful with backends like Google Drive which
|
||||
can have duplicate file names. It can be run on wrapping backends
|
||||
(e.g. crypt) if they wrap a backend which supports duplicate file
|
||||
names.
|
||||
|
||||
|
||||
@@ -29,15 +29,15 @@ is an **empty** **existing** directory:
|
||||
|
||||
On Windows you can start a mount in different ways. See [below](#mounting-modes-on-windows)
|
||||
for details. The following examples will mount to an automatically assigned drive,
|
||||
to specific drive letter `X:`, to path `C:\path\parent\mount`
|
||||
(where parent directory or drive must exist, and mount must **not** exist,
|
||||
to specific drive letter `X:`, to path `C:\path\to\nonexistent\directory`
|
||||
(which must be **non-existent** subdirectory of an **existing** parent directory or drive,
|
||||
and is not supported when [mounting as a network drive](#mounting-modes-on-windows)), and
|
||||
the last example will mount as network share `\\cloud\remote` and map it to an
|
||||
automatically assigned drive:
|
||||
|
||||
rclone mount remote:path/to/files *
|
||||
rclone mount remote:path/to/files X:
|
||||
rclone mount remote:path/to/files C:\path\parent\mount
|
||||
rclone mount remote:path/to/files C:\path\to\nonexistent\directory
|
||||
rclone mount remote:path/to/files \\cloud\remote
|
||||
|
||||
When the program ends while in foreground mode, either via Ctrl+C or receiving
|
||||
@@ -91,14 +91,14 @@ and experience unexpected program errors, freezes or other issues, consider moun
|
||||
as a network drive instead.
|
||||
|
||||
When mounting as a fixed disk drive you can either mount to an unused drive letter,
|
||||
or to a path representing a **non-existent** subdirectory of an **existing** parent
|
||||
or to a path - which must be **non-existent** subdirectory of an **existing** parent
|
||||
directory or drive. Using the special value `*` will tell rclone to
|
||||
automatically assign the next available drive letter, starting with Z: and moving backward.
|
||||
Examples:
|
||||
|
||||
rclone mount remote:path/to/files *
|
||||
rclone mount remote:path/to/files X:
|
||||
rclone mount remote:path/to/files C:\path\parent\mount
|
||||
rclone mount remote:path/to/files C:\path\to\nonexistent\directory
|
||||
rclone mount remote:path/to/files X:
|
||||
|
||||
Option `--volname` can be used to set a custom volume name for the mounted
|
||||
@@ -171,24 +171,10 @@ Note that the mapping of permissions is not always trivial, and the result
|
||||
you see in Windows Explorer may not be exactly like you expected.
|
||||
For example, when setting a value that includes write access, this will be
|
||||
mapped to individual permissions "write attributes", "write data" and "append data",
|
||||
but not "write extended attributes". Windows will then show this as basic
|
||||
permission "Special" instead of "Write", because "Write" includes the
|
||||
"write extended attributes" permission.
|
||||
|
||||
If you set POSIX permissions for only allowing access to the owner, using
|
||||
`--file-perms 0600 --dir-perms 0700`, the user group and the built-in "Everyone"
|
||||
group will still be given some special permissions, such as "read attributes"
|
||||
and "read permissions", in Windows. This is done for compatibility reasons,
|
||||
e.g. to allow users without additional permissions to be able to read basic
|
||||
metadata about files like in UNIX. One case that may arise is that other programs
|
||||
(incorrectly) interprets this as the file being accessible by everyone. For example
|
||||
an SSH client may warn about "unprotected private key file".
|
||||
|
||||
WinFsp 2021 (version 1.9, still in beta) introduces a new FUSE option "FileSecurity",
|
||||
that allows the complete specification of file security descriptors using
|
||||
[SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
|
||||
With this you can work around issues such as the mentioned "unprotected private key file"
|
||||
by specifying `-o FileSecurity="D:P(A;;FA;;;OW)"`, for file all access (FA) to the owner (OW).
|
||||
but not "write extended attributes" (WinFsp does not support extended attributes,
|
||||
see [this](https://github.com/billziss-gh/winfsp/wiki/NTFS-Compatibility)).
|
||||
Windows will then show this as basic permission "Special" instead of "Write",
|
||||
because "Write" includes the "write extended attributes" permission.
|
||||
|
||||
### Windows caveats
|
||||
|
||||
@@ -392,13 +378,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -542,19 +521,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
|
||||
```
|
||||
rclone mount remote:path /path/to/mountpoint [flags]
|
||||
@@ -599,7 +565,6 @@ rclone mount remote:path /path/to/mountpoint [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
--volname string Set the volume name. Supported on Windows and OSX only.
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
---
|
||||
title: "rclone selfupdate"
|
||||
description: "Update the rclone binary."
|
||||
slug: rclone_selfupdate
|
||||
url: /commands/rclone_selfupdate/
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/selfupdate/ and as part of making a release run "make commanddocs"
|
||||
---
|
||||
# rclone selfupdate
|
||||
|
||||
Update the rclone binary.
|
||||
|
||||
## Synopsis
|
||||
|
||||
|
||||
This command downloads the latest release of rclone and replaces
|
||||
the currently running binary. The download is verified with a hashsum
|
||||
and cryptographically signed signature.
|
||||
|
||||
If used without flags (or with implied `--stable` flag), this command
|
||||
will install the latest stable release. However, some issues may be fixed
|
||||
(or features added) only in the latest beta release. In such cases you should
|
||||
run the command with the `--beta` flag, i.e. `rclone selfupdate --beta`.
|
||||
You can check in advance what version would be installed by adding the
|
||||
`--check` flag, then repeat the command without it when you are satisfied.
|
||||
|
||||
Sometimes the rclone team may recommend you a concrete beta or stable
|
||||
rclone release to troubleshoot your issue or add a bleeding edge feature.
|
||||
The `--version VER` flag, if given, will update to the concrete version
|
||||
instead of the latest one. If you omit micro version from `VER` (for
|
||||
example `1.53`), the latest matching micro version will be used.
|
||||
|
||||
Upon successful update rclone will print a message that contains a previous
|
||||
version number. You will need it if you later decide to revert your update
|
||||
for some reason. Then you'll have to note the previous version and run the
|
||||
following command: `rclone selfupdate [--beta] OLDVER`.
|
||||
If the old version contains only dots and digits (for example `v1.54.0`)
|
||||
then it's a stable release so you won't need the `--beta` flag. Beta releases
|
||||
have an additional information similar to `v1.54.0-beta.5111.06f1c0c61`.
|
||||
(if you are a developer and use a locally built rclone, the version number
|
||||
will end with `-DEV`, you will have to rebuild it as it obvisously can't
|
||||
be distributed).
|
||||
|
||||
If you previously installed rclone via a package manager, the package may
|
||||
include local documentation or configure services. You may wish to update
|
||||
with the flag `--package deb` or `--package rpm` (whichever is correct for
|
||||
your OS) to update these too. This command with the default `--package zip`
|
||||
will update only the rclone executable so the local manual may become
|
||||
inaccurate after it.
|
||||
|
||||
The `rclone mount` command (https://rclone.org/commands/rclone_mount/) may
|
||||
or may not support extended FUSE options depending on the build and OS.
|
||||
`selfupdate` will refuse to update if the capability would be discarded.
|
||||
|
||||
Note: Windows forbids deletion of a currently running executable so this
|
||||
command will rename the old executable to 'rclone.old.exe' upon success.
|
||||
|
||||
Please note that this command was not available before rclone version 1.55.
|
||||
If it fails for you with the message `unknown command "selfupdate"` then
|
||||
you will need to update manually following the install instructions located
|
||||
at https://rclone.org/install/
|
||||
|
||||
|
||||
```
|
||||
rclone selfupdate [flags]
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
```
|
||||
--beta Install beta release.
|
||||
--check Check for latest release, do not download.
|
||||
-h, --help help for selfupdate
|
||||
--output string Save the downloaded binary at a given path (default: replace running binary)
|
||||
--package string Package format: zip|deb|rpm (default: zip)
|
||||
--stable Install stable release (this is the default)
|
||||
--version string Install the given rclone version (default: latest)
|
||||
```
|
||||
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
## SEE ALSO
|
||||
|
||||
* [rclone](/commands/rclone/) - Show help for rclone commands, flags and backends.
|
||||
|
||||
@@ -134,13 +134,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -284,19 +277,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
|
||||
```
|
||||
rclone serve dlna remote:path [flags]
|
||||
@@ -329,7 +309,6 @@ rclone serve dlna remote:path [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
```
|
||||
|
||||
@@ -133,13 +133,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -283,19 +276,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
## Auth Proxy
|
||||
|
||||
If you supply the parameter `--auth-proxy /path/to/program` then
|
||||
@@ -414,7 +394,6 @@ rclone serve ftp remote:path [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
```
|
||||
|
||||
@@ -205,13 +205,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -355,19 +348,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
|
||||
```
|
||||
rclone serve http remote:path [flags]
|
||||
@@ -410,7 +390,6 @@ rclone serve http remote:path [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
```
|
||||
|
||||
@@ -144,13 +144,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -294,19 +287,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
## Auth Proxy
|
||||
|
||||
If you supply the parameter `--auth-proxy /path/to/program` then
|
||||
@@ -424,7 +404,6 @@ rclone serve sftp remote:path [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
```
|
||||
|
||||
@@ -213,13 +213,6 @@ for two reasons. Firstly because it is only checked every
|
||||
`--vfs-cache-poll-interval`. Secondly because open files cannot be
|
||||
evicted from the cache.
|
||||
|
||||
You **should not** run two copies of rclone using the same VFS cache
|
||||
with the same or overlapping remotes if using `--vfs-cache-mode > off`.
|
||||
This can potentially cause data corruption if you do. You can work
|
||||
around this by giving each rclone its own cache hierarchy with
|
||||
`--cache-dir`. You don't need to worry about this if the remotes in
|
||||
use don't overlap.
|
||||
|
||||
### --vfs-cache-mode off
|
||||
|
||||
In this mode (the default) the cache will read directly from the remote and write
|
||||
@@ -363,19 +356,6 @@ If the flag is not provided on the command line, then its default value depends
|
||||
on the operating system where rclone runs: "true" on Windows and macOS, "false"
|
||||
otherwise. If the flag is provided without a value, then it is "true".
|
||||
|
||||
## Alternate report of used bytes
|
||||
|
||||
Some backends, most notably S3, do not report the amount of bytes used.
|
||||
If you need this information to be available when running `df` on the
|
||||
filesystem, then pass the flag `--vfs-used-is-size` to rclone.
|
||||
With this flag set, instead of relying on the backend to report this
|
||||
information, rclone will scan the whole remote similar to `rclone size`
|
||||
and compute the total used space itself.
|
||||
|
||||
_WARNING._ Contrary to `rclone size`, this flag ignores filters so that the
|
||||
result is accurate. However, this is very inefficient and may cost lots of API
|
||||
calls resulting in extra charges. Use it as a last resort and only with caching.
|
||||
|
||||
## Auth Proxy
|
||||
|
||||
If you supply the parameter `--auth-proxy /path/to/program` then
|
||||
@@ -502,7 +482,6 @@ rclone serve webdav remote:path [flags]
|
||||
--vfs-read-chunk-size SizeSuffix Read the source objects in chunks. (default 128M)
|
||||
--vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. 'off' is unlimited. (default off)
|
||||
--vfs-read-wait duration Time to wait for in-sequence read before seeking. (default 20ms)
|
||||
--vfs-used-is-size rclone size Use the rclone size algorithm for Used size.
|
||||
--vfs-write-back duration Time to writeback files after last use when using cache. (default 5s)
|
||||
--vfs-write-wait duration Time to wait for in-sequence write before giving error. (default 1s)
|
||||
```
|
||||
|
||||
@@ -15,8 +15,7 @@ Make source and dest identical, modifying destination only.
|
||||
Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer unchanged files, testing by size and
|
||||
modification time or MD5SUM. Destination is updated to match
|
||||
source, including deleting files if necessary (except duplicate
|
||||
objects, see below).
|
||||
source, including deleting files if necessary.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
`--dry-run` or the `--interactive`/`-i` flag.
|
||||
@@ -24,8 +23,7 @@ objects, see below).
|
||||
rclone sync -i SOURCE remote:DESTINATION
|
||||
|
||||
Note that files in the destination won't be deleted if there were any
|
||||
errors at any point. Duplicate objects (files with the same name, on
|
||||
those providers that support it) are also not yet handled.
|
||||
errors at any point.
|
||||
|
||||
It is always the contents of the directory that is synced, not the
|
||||
directory so when source:path is a directory, it's the contents of
|
||||
@@ -37,9 +35,6 @@ go there.
|
||||
|
||||
**Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
|
||||
|
||||
```
|
||||
rclone sync source:path dest:path [flags]
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
---
|
||||
title: "rclone test"
|
||||
description: "Run a test command"
|
||||
slug: rclone_test
|
||||
url: /commands/rclone_test/
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/ and as part of making a release run "make commanddocs"
|
||||
---
|
||||
# rclone test
|
||||
|
||||
Run a test command
|
||||
|
||||
## Synopsis
|
||||
|
||||
Rclone test is used to run test commands.
|
||||
|
||||
Select which test comand you want with the subcommand, eg
|
||||
|
||||
rclone test memory remote:
|
||||
|
||||
Each subcommand has its own options which you can see in their help.
|
||||
|
||||
**NB** Be careful running these commands, they may do strange things
|
||||
so reading their documentation first is recommended.
|
||||
|
||||
|
||||
## Options
|
||||
|
||||
```
|
||||
-h, --help help for test
|
||||
```
|
||||
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
## SEE ALSO
|
||||
|
||||
* [rclone](/commands/rclone/) - Show help for rclone commands, flags and backends.
|
||||
* [rclone test histogram](/commands/rclone_test_histogram/) - Makes a histogram of file name characters.
|
||||
* [rclone test info](/commands/rclone_test_info/) - Discovers file name or other limitations for paths.
|
||||
* [rclone test makefiles](/commands/rclone_test_makefiles/) - Make a random file hierarchy in <dir>
|
||||
* [rclone test memory](/commands/rclone_test_memory/) - Load all the objects at remote:path into memory and report memory stats.
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
---
|
||||
title: "rclone test histogram"
|
||||
description: "Makes a histogram of file name characters."
|
||||
slug: rclone_test_histogram
|
||||
url: /commands/rclone_test_histogram/
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/histogram/ and as part of making a release run "make commanddocs"
|
||||
---
|
||||
# rclone test histogram
|
||||
|
||||
Makes a histogram of file name characters.
|
||||
|
||||
## Synopsis
|
||||
|
||||
This command outputs JSON which shows the histogram of characters used
|
||||
in filenames in the remote:path specified.
|
||||
|
||||
The data doesn't contain any identifying information but is useful for
|
||||
the rclone developers when developing filename compression.
|
||||
|
||||
|
||||
```
|
||||
rclone test histogram [remote:path] [flags]
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
```
|
||||
-h, --help help for histogram
|
||||
```
|
||||
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
## SEE ALSO
|
||||
|
||||
* [rclone test](/commands/rclone_test/) - Run a test command
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
---
|
||||
title: "rclone test info"
|
||||
description: "Discovers file name or other limitations for paths."
|
||||
slug: rclone_test_info
|
||||
url: /commands/rclone_test_info/
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/info/ and as part of making a release run "make commanddocs"
|
||||
---
|
||||
# rclone test info
|
||||
|
||||
Discovers file name or other limitations for paths.
|
||||
|
||||
## Synopsis
|
||||
|
||||
rclone info discovers what filenames and upload methods are possible
|
||||
to write to the paths passed in and how long they can be. It can take some
|
||||
time. It will write test files into the remote:path passed in. It outputs
|
||||
a bit of go code for each one.
|
||||
|
||||
**NB** this can create undeletable files and other hazards - use with care
|
||||
|
||||
|
||||
```
|
||||
rclone test info [remote:path]+ [flags]
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
```
|
||||
--all Run all tests.
|
||||
--check-control Check control characters.
|
||||
--check-length Check max filename length.
|
||||
--check-normalization Check UTF-8 Normalization.
|
||||
--check-streaming Check uploads with indeterminate file size.
|
||||
-h, --help help for info
|
||||
--upload-wait duration Wait after writing a file.
|
||||
--write-json string Write results to file.
|
||||
```
|
||||
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
## SEE ALSO
|
||||
|
||||
* [rclone test](/commands/rclone_test/) - Run a test command
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user