jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-06 00:03:32 +00:00
Code Issues Projects Releases Wiki Activity

Fix excessive retries missing --max-duration timeout - fixes #4504

Browse Source 
This change checks the context whenever rclone might retry, and
doesn't retry if the current context has an error.

This fixes the pathological behaviour of `--max-duration` refusing to
exit because all the context deadline exceeded errors were being
retried.

This unfortunately meant changing the shouldRetry logic in every
backend and doing a lot of context propagation.

See: https://forum.rclone.org/t/add-flag-to-exit-immediately-when-max-duration-reached/22723
This commit is contained in:
Nick Craig-Wood
2021-03-11 14:44:01 +00:00
parent 32925dae1f
commit 4013bc4a4c
31 changed files with 474 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
backend/dropbox/dropbox.go
View File

@@ -292,7 +292,10 @@ func (f *Fs) Features() *fs.Features {
// shouldRetry returns a boolean as to whether this err deserves to be
// retried. It returns the err as a convenience
func shouldRetry(err error) (bool, error) {
func shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if err == nil {
return false, err
}
@@ -425,7 +428,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if f.root == "" {
return f, nil
}
_, err := f.findSharedFile(f.root)
_, err := f.findSharedFile(ctx, f.root)
f.root = ""
if err == nil {
return f, fs.ErrorIsFile
@@ -445,7 +448,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// root is not empty so we have find the right shared folder if it exists
id, err := f.findSharedFolder(dir)
id, err := f.findSharedFolder(ctx, dir)
if err != nil {
// if we didn't find the specified shared folder we have to bail out here
return nil, err
@@ -453,7 +456,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// we found the specified shared folder so let's mount it
// this will add it to the users normal root namespace and allows us
// to actually perform operations on it using the normal api endpoints.
err = f.mountSharedFolder(id)
err = f.mountSharedFolder(ctx, id)
if err != nil {
switch e := err.(type) {
case sharing.MountFolderAPIError:
@@ -477,7 +480,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
var acc *users.FullAccount
err = f.pacer.Call(func() (bool, error) {
acc, err = f.users.GetCurrentAccount()
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "get current account failed")
@@ -495,7 +498,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.setRoot(root)
// See if the root is actually an object
_, err = f.getFileMetadata(f.slashRoot)
_, err = f.getFileMetadata(ctx, f.slashRoot)
if err == nil {
newRoot := path.Dir(f.root)
if newRoot == "." {
@@ -529,12 +532,12 @@ func (f *Fs) setRoot(root string) {
}
// getMetadata gets the metadata for a file or directory
func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
err = f.pacer.Call(func() (bool, error) {
entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
Path: f.opt.Enc.FromStandardPath(objPath),
})
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
switch e := err.(type) {
@@ -549,8 +552,8 @@ func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool,
}
// getFileMetadata gets the metadata for a file
func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
entry, notFound, err := f.getMetadata(filePath)
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
entry, notFound, err := f.getMetadata(ctx, filePath)
if err != nil {
return nil, err
}
@@ -565,8 +568,8 @@ func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err
}
// getDirMetadata gets the metadata for a directory
func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
entry, notFound, err := f.getMetadata(dirPath)
func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
entry, notFound, err := f.getMetadata(ctx, dirPath)
if err != nil {
return nil, err
}
@@ -583,7 +586,7 @@ func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *files.FileMetadata) (fs.Object, error) {
o := &Object{
fs: f,
remote: remote,
@@ -592,7 +595,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
if info != nil {
err = o.setMetadataFromEntry(info)
} else {
err = o.readEntryAndSetMetadata()
err = o.readEntryAndSetMetadata(ctx)
}
if err != nil {
return nil, err
@@ -604,14 +607,14 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if f.opt.SharedFiles {
return f.findSharedFile(remote)
return f.findSharedFile(ctx, remote)
}
return f.newObjectWithInfo(remote, nil)
return f.newObjectWithInfo(ctx, remote, nil)
}
// listSharedFoldersApi lists all available shared folders mounted and not mounted
// we'll need the id later so we have to return them in original format
func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
started := false
var res *sharing.ListFoldersResult
for {
@@ -621,7 +624,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFolders(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
@@ -633,7 +636,7 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContinue(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "list continue")
@@ -658,8 +661,8 @@ func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
// findSharedFolder find the id for a given shared folder name
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
// so our only option is to iterate over all shared folders
func (f *Fs) findSharedFolder(name string) (id string, err error) {
entries, err := f.listSharedFolders()
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
entries, err := f.listSharedFolders(ctx)
if err != nil {
return "", err
}
@@ -672,20 +675,20 @@ func (f *Fs) findSharedFolder(name string) (id string, err error) {
}
// mountSharedFolder mount a shared folder to the root namespace
func (f *Fs) mountSharedFolder(id string) error {
func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
arg := sharing.MountFolderArg{
SharedFolderId: id,
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.sharing.MountFolder(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
return err
}
// listReceivedFiles lists shared the user as access to (note this means individual
// files not files contained in shared folders)
func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {
started := false
var res *sharing.ListFilesResult
for {
@@ -695,7 +698,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFiles(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
@@ -707,7 +710,7 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContinue(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "list continue")
@@ -734,8 +737,8 @@ func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
return entries, nil
}
func (f *Fs) findSharedFile(name string) (o *Object, err error) {
files, err := f.listReceivedFiles()
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
files, err := f.listReceivedFiles(ctx)
if err != nil {
return nil, err
}
@@ -758,10 +761,10 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
if f.opt.SharedFiles {
return f.listReceivedFiles()
return f.listReceivedFiles(ctx)
}
if f.opt.SharedFolders {
return f.listSharedFolders()
return f.listSharedFolders(ctx)
}
root := f.slashRoot
@@ -782,7 +785,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
switch e := err.(type) {
@@ -800,7 +803,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "list continue")
@@ -830,7 +833,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
entries = append(entries, d)
} else if fileInfo != nil {
o, err := f.newObjectWithInfo(remote, fileInfo)
o, err := f.newObjectWithInfo(ctx, remote, fileInfo)
if err != nil {
return nil, err
}
@@ -879,7 +882,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
}
// check directory doesn't exist
_, err := f.getDirMetadata(root)
_, err := f.getDirMetadata(ctx, root)
if err == nil {
return nil // directory exists already
} else if err != fs.ErrorDirNotFound {
@@ -896,7 +899,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.CreateFolderV2(&arg2)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
return err
}
@@ -913,7 +916,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
if check {
// check directory exists
_, err = f.getDirMetadata(root)
_, err = f.getDirMetadata(ctx, root)
if err != nil {
return errors.Wrap(err, "Rmdir")
}
@@ -930,7 +933,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
var res *files.ListFolderResult
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return errors.Wrap(err, "Rmdir")
@@ -943,7 +946,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
// remove it
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
return shouldRetry(err)
return shouldRetry(ctx, err)
})
return err
}
@@ -996,7 +999,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.CopyV2(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "copy failed")
@@ -1057,7 +1060,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.MoveV2(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "move failed")
@@ -1091,7 +1094,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var linkRes sharing.IsSharedLinkMetadata
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil && strings.Contains(err.Error(),
@@ -1104,7 +1107,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
var listRes *sharing.ListSharedLinksResult
err = f.pacer.Call(func() (bool, error) {
listRes, err = f.sharing.ListSharedLinks(&listArg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return
@@ -1146,7 +1149,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := path.Join(f.slashRoot, dstRemote)
// Check if destination exists
_, err := f.getDirMetadata(dstPath)
_, err := f.getDirMetadata(ctx, dstPath)
if err == nil {
return fs.ErrorDirExists
} else if err != fs.ErrorDirNotFound {
@@ -1165,7 +1168,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.MoveV2(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return errors.Wrap(err, "MoveDir failed")
@@ -1179,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var q *users.SpaceUsage
err = f.pacer.Call(func() (bool, error) {
q, err = f.users.GetSpaceUsage()
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, errors.Wrap(err, "about failed")
@@ -1210,7 +1213,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
go func() {
// get the StartCursor early so all changes from now on get processed
startCursor, err := f.changeNotifyCursor()
startCursor, err := f.changeNotifyCursor(ctx)
if err != nil {
fs.Infof(f, "Failed to get StartCursor: %s", err)
}
@@ -1235,7 +1238,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}
case <-tickerC:
if startCursor == "" {
startCursor, err = f.changeNotifyCursor()
startCursor, err = f.changeNotifyCursor(ctx)
if err != nil {
fs.Infof(f, "Failed to get StartCursor: %s", err)
continue
@@ -1251,7 +1254,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}()
}
func (f *Fs) changeNotifyCursor() (cursor string, err error) {
func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error) {
var startCursor *files.ListFolderGetLatestCursorResult
err = f.pacer.Call(func() (bool, error) {
@@ -1266,7 +1269,7 @@ func (f *Fs) changeNotifyCursor() (cursor string, err error) {
startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return
@@ -1296,7 +1299,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
}
res, err = f.svc.ListFolderLongpoll(&args)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return
@@ -1319,7 +1322,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
}
err = f.pacer.Call(func() (bool, error) {
changeList, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return "", errors.Wrap(err, "list continue")
@@ -1392,7 +1395,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != DbHashType {
return "", hash.ErrUnsupported
}
err := o.readMetaData()
err := o.readMetaData(ctx)
if err != nil {
return "", errors.Wrap(err, "failed to read hash from metadata")
}
@@ -1416,17 +1419,17 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
}
// Reads the entry for a file from dropbox
func (o *Object) readEntry() (*files.FileMetadata, error) {
return o.fs.getFileMetadata(o.remotePath())
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
return o.fs.getFileMetadata(ctx, o.remotePath())
}
// Read entry if not set and set metadata from it
func (o *Object) readEntryAndSetMetadata() error {
func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
// Last resort set time from client
if !o.modTime.IsZero() {
return nil
}
entry, err := o.readEntry()
entry, err := o.readEntry(ctx)
if err != nil {
return err
}
@@ -1439,12 +1442,12 @@ func (o *Object) remotePath() string {
}
// readMetaData gets the info if it hasn't already been fetched
func (o *Object) readMetaData() (err error) {
func (o *Object) readMetaData(ctx context.Context) (err error) {
if !o.modTime.IsZero() {
return nil
}
// Last resort
return o.readEntryAndSetMetadata()
return o.readEntryAndSetMetadata(ctx)
}
// ModTime returns the modification time of the object
@@ -1452,7 +1455,7 @@ func (o *Object) readMetaData() (err error) {
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData()
err := o.readMetaData(ctx)
if err != nil {
fs.Debugf(o, "Failed to read metadata: %v", err)
return time.Now()
@@ -1486,7 +1489,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
@@ -1502,7 +1505,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.srv.Download(&arg)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
switch e := err.(type) {
@@ -1521,7 +1524,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// Will work optimally if size is >= uploadChunkSize. If the size is either
// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
// avoidable request to the Dropbox API that does not carry payload.
func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
chunkSize := int64(o.fs.opt.ChunkSize)
chunks := 0
if size != -1 {
@@ -1550,7 +1553,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
return false, nil
}
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
@@ -1676,11 +1679,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var err error
var entry *files.FileMetadata
if size > int64(o.fs.opt.ChunkSize) || size == -1 {
entry, err = o.uploadChunked(in, commitInfo, size)
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
} else {
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
entry, err = o.fs.srv.Upload(commitInfo, in)
return shouldRetry(err)
return shouldRetry(ctx, err)
})
}
if err != nil {
@@ -1698,7 +1701,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
})
return shouldRetry(err)
return shouldRetry(ctx, err)
})
return err
}