jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-01-08 03:23:26 +00:00
Code Issues Projects Releases Wiki Activity

Add context propagation to rclone

Browse Source 
- Change rclone/fs interfaces to accept context.Context
- Update interface implementations to use context.Context
- Change top level usage to propagate context to lover level functions

Context propagation is needed for stopping transfers and passing other
request-scoped values.
This commit is contained in:
Aleksandar Jankovic
2019-06-17 10:34:30 +02:00
committed by Nick Craig-Wood
parent a2c317b46e
commit f78cd1e043
156 changed files with 2570 additions and 2380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fs/accounting/stats.go
View File

@@ -2,6 +2,7 @@ package accounting
import (
"bytes"
"context"
"fmt"
"strings"
"sync"
@@ -101,7 +102,7 @@ func NewStats() *StatsInfo {
}
// RemoteStats returns stats for rc
func (s *StatsInfo) RemoteStats(in rc.Params) (out rc.Params, err error) {
func (s *StatsInfo) RemoteStats(ctx context.Context, in rc.Params) (out rc.Params, err error) {
out = make(rc.Params)
s.mu.RLock()
dt := time.Now().Sub(s.start)

2
fs/accounting/token_bucket.go
View File

@@ -132,7 +132,7 @@ func SetBwLimit(bandwidth fs.SizeSuffix) {
func init() {
rc.Add(rc.Call{
Path: "core/bwlimit",
Fn: func(in rc.Params) (out rc.Params, err error) {
Fn: func(ctx context.Context, in rc.Params) (out rc.Params, err error) {
ibwlimit, ok := in["rate"]
if !ok {
return out, errors.Errorf("parameter rate not found")

17
fs/chunkedreader/chunkedreader.go
View File

@@ -1,6 +1,7 @@
package chunkedreader
import (
"context"
"errors"
"io"
"sync"
@@ -19,6 +20,7 @@ var (
//
// A initialChunkSize of <= 0 will disable chunked reading.
type ChunkedReader struct {
ctx context.Context
mu sync.Mutex // protects following fields
o fs.Object // source to read from
rc io.ReadCloser // reader for the current open chunk
@@ -37,7 +39,7 @@ type ChunkedReader struct {
// If maxChunkSize is greater than initialChunkSize, the chunk size will be
// doubled after each chunk read with a maximun of maxChunkSize.
// A Seek or RangeSeek will reset the chunk size to it's initial value
func New(o fs.Object, initialChunkSize int64, maxChunkSize int64) *ChunkedReader {
func New(ctx context.Context, o fs.Object, initialChunkSize int64, maxChunkSize int64) *ChunkedReader {
if initialChunkSize <= 0 {
initialChunkSize = -1
}
@@ -45,6 +47,7 @@ func New(o fs.Object, initialChunkSize int64, maxChunkSize int64) *ChunkedReader
maxChunkSize = initialChunkSize
}
return &ChunkedReader{
ctx: ctx,
o: o,
offset: -1,
chunkSize: initialChunkSize,
@@ -129,14 +132,14 @@ func (cr *ChunkedReader) Close() error {
// Seek the file - for details see io.Seeker
func (cr *ChunkedReader) Seek(offset int64, whence int) (int64, error) {
return cr.RangeSeek(offset, whence, -1)
return cr.RangeSeek(context.TODO(), offset, whence, -1)
}
// RangeSeek the file - for details see RangeSeeker
//
// The specified length will only apply to the next chunk opened.
// RangeSeek will not reopen the source until Read is called.
func (cr *ChunkedReader) RangeSeek(offset int64, whence int, length int64) (int64, error) {
func (cr *ChunkedReader) RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error) {
cr.mu.Lock()
defer cr.mu.Unlock()
@@ -196,7 +199,7 @@ func (cr *ChunkedReader) openRange() error {
}
if rs, ok := cr.rc.(fs.RangeSeeker); ok {
n, err := rs.RangeSeek(offset, io.SeekStart, length)
n, err := rs.RangeSeek(cr.ctx, offset, io.SeekStart, length)
if err == nil && n == offset {
cr.offset = offset
return nil
@@ -212,12 +215,12 @@ func (cr *ChunkedReader) openRange() error {
var err error
if length <= 0 {
if offset == 0 {
rc, err = cr.o.Open()
rc, err = cr.o.Open(cr.ctx)
} else {
rc, err = cr.o.Open(&fs.RangeOption{Start: offset, End: -1})
rc, err = cr.o.Open(cr.ctx, &fs.RangeOption{Start: offset, End: -1})
}
} else {
rc, err = cr.o.Open(&fs.RangeOption{Start: offset, End: offset + length - 1})
rc, err = cr.o.Open(cr.ctx, &fs.RangeOption{Start: offset, End: offset + length - 1})
}
if err != nil {
return err

15
fs/chunkedreader/chunkedreader_test.go
View File

@@ -1,6 +1,7 @@
package chunkedreader
import (
"context"
"fmt"
"io"
"math/rand"
@@ -38,13 +39,13 @@ func testRead(content []byte, mode mockobject.SeekMode) func(*testing.T) {
}
t.Run(fmt.Sprintf("Chunksize_%d_%d", cs, csMax), func(t *testing.T) {
cr := New(o, cs, csMax)
cr := New(context.Background(), o, cs, csMax)
for _, offset := range offsets {
for _, limit := range limits {
what := fmt.Sprintf("offset %d, limit %d", offset, limit)
p, err := cr.RangeSeek(offset, io.SeekStart, limit)
p, err := cr.RangeSeek(context.Background(), offset, io.SeekStart, limit)
if offset >= cl {
require.Error(t, err, what)
return
@@ -78,27 +79,27 @@ func TestErrorAfterClose(t *testing.T) {
o := mockobject.New("test.bin").WithContent(content, mockobject.SeekModeNone)
// Close
cr := New(o, 0, 0)
cr := New(context.Background(), o, 0, 0)
require.NoError(t, cr.Close())
require.Error(t, cr.Close())
// Read
cr = New(o, 0, 0)
cr = New(context.Background(), o, 0, 0)
require.NoError(t, cr.Close())
var buf [1]byte
_, err := cr.Read(buf[:])
require.Error(t, err)
// Seek
cr = New(o, 0, 0)
cr = New(context.Background(), o, 0, 0)
require.NoError(t, cr.Close())
_, err = cr.Seek(1, io.SeekCurrent)
require.Error(t, err)
// RangeSeek
cr = New(o, 0, 0)
cr = New(context.Background(), o, 0, 0)
require.NoError(t, cr.Close())
_, err = cr.RangeSeek(1, io.SeekCurrent, 0)
_, err = cr.RangeSeek(context.Background(), 1, io.SeekCurrent, 0)
require.Error(t, err)
}

18
fs/config/rc.go
View File

@@ -1,6 +1,8 @@
package config
import (
"context"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/rc"
)
@@ -23,7 +25,7 @@ See the [config dump command](/commands/rclone_config_dump/) command for more in
}
// Return the config file dump
func rcDump(in rc.Params) (out rc.Params, err error) {
func rcDump(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return DumpRcBlob(), nil
}
@@ -43,7 +45,7 @@ See the [config dump command](/commands/rclone_config_dump/) command for more in
}
// Return the config file get
func rcGet(in rc.Params) (out rc.Params, err error) {
func rcGet(ctx context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
@@ -67,7 +69,7 @@ See the [listremotes command](/commands/rclone_listremotes/) command for more in
}
// Return the a list of remotes in the config file
func rcListRemotes(in rc.Params) (out rc.Params, err error) {
func rcListRemotes(ctx context.Context, in rc.Params) (out rc.Params, err error) {
var remotes = []string{}
for _, remote := range getConfigData().GetSectionList() {
remotes = append(remotes, remote)
@@ -94,7 +96,7 @@ See the [config providers command](/commands/rclone_config_providers/) command f
}
// Return the config file providers
func rcProviders(in rc.Params) (out rc.Params, err error) {
func rcProviders(ctx context.Context, in rc.Params) (out rc.Params, err error) {
out = rc.Params{
"providers": fs.Registry,
}
@@ -111,8 +113,8 @@ func init() {
rc.Add(rc.Call{
Path: "config/" + name,
AuthRequired: true,
Fn: func(in rc.Params) (rc.Params, error) {
return rcConfig(in, name)
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
return rcConfig(ctx, in, name)
},
Title: name + " the config for a remote.",
Help: `This takes the following parameters
@@ -126,7 +128,7 @@ See the [config ` + name + ` command](/commands/rclone_config_` + name + `/) com
}
// Manipulate the config file
func rcConfig(in rc.Params, what string) (out rc.Params, err error) {
func rcConfig(ctx context.Context, in rc.Params, what string) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err
@@ -167,7 +169,7 @@ See the [config delete command](/commands/rclone_config_delete/) command for mor
}
// Return the config file delete
func rcDelete(in rc.Params) (out rc.Params, err error) {
func rcDelete(ctx context.Context, in rc.Params) (out rc.Params, err error) {
name, err := in.GetString("name")
if err != nil {
return nil, err

17
fs/config/rc_test.go
View File

@@ -1,6 +1,7 @@
package config
import (
"context"
"testing"
_ "github.com/ncw/rclone/backend/local"
@@ -24,7 +25,7 @@ func TestRc(t *testing.T) {
"test_key": "sausage",
},
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.Nil(t, out)
assert.Equal(t, "local", FileGet(testName, "type"))
@@ -37,7 +38,7 @@ func TestRc(t *testing.T) {
call := rc.Calls.Get("config/dump")
assert.NotNil(t, call)
in := rc.Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
@@ -54,7 +55,7 @@ func TestRc(t *testing.T) {
in := rc.Params{
"name": testName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
@@ -66,7 +67,7 @@ func TestRc(t *testing.T) {
call := rc.Calls.Get("config/listremotes")
assert.NotNil(t, call)
in := rc.Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
@@ -87,7 +88,7 @@ func TestRc(t *testing.T) {
"test_key2": "cabbage",
},
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Nil(t, out)
@@ -106,7 +107,7 @@ func TestRc(t *testing.T) {
"test_key2": "cabbage",
},
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Nil(t, out)
@@ -121,7 +122,7 @@ func TestRc(t *testing.T) {
in = rc.Params{
"name": testName,
}
out, err = call.Fn(in)
out, err = call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Nil(t, out)
assert.Equal(t, "", FileGet(testName, "type"))
@@ -132,7 +133,7 @@ func TestRcProviders(t *testing.T) {
call := rc.Calls.Get("config/providers")
assert.NotNil(t, call)
in := rc.Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
var registry []*fs.RegInfo

11
fs/dir.go
View File

@@ -1,6 +1,9 @@
package fs
import "time"
import (
"context"
"time"
)
// Dir describes an unspecialized directory for directory/container/bucket lists
type Dir struct {
@@ -22,10 +25,10 @@ func NewDir(remote string, modTime time.Time) *Dir {
}
// NewDirCopy creates an unspecialized copy of the Directory object passed in
func NewDirCopy(d Directory) *Dir {
func NewDirCopy(ctx context.Context, d Directory) *Dir {
return &Dir{
remote: d.Remote(),
modTime: d.ModTime(),
modTime: d.ModTime(ctx),
size: d.Size(),
items: d.Items(),
id: d.ID(),
@@ -61,7 +64,7 @@ func (d *Dir) SetID(id string) *Dir {
// ModTime returns the modification date of the file
// It should return a best guess if one isn't available
func (d *Dir) ModTime() time.Time {
func (d *Dir) ModTime(ctx context.Context) time.Time {
if !d.modTime.IsZero() {
return d.modTime
}

19
fs/filter/filter.go
View File

@@ -3,6 +3,7 @@ package filter
import (
"bufio"
"context"
"fmt"
"log"
"os"
@@ -399,12 +400,12 @@ func (f *Filter) ListContainsExcludeFile(entries fs.DirEntries) bool {
// IncludeDirectory returns a function which checks whether this
// directory should be included in the sync or not.
func (f *Filter) IncludeDirectory(fs fs.Fs) func(string) (bool, error) {
func (f *Filter) IncludeDirectory(ctx context.Context, fs fs.Fs) func(string) (bool, error) {
return func(remote string) (bool, error) {
remote = strings.Trim(remote, "/")
// first check if we need to remove directory based on
// the exclude file
excl, err := f.DirContainsExcludeFile(fs, remote)
excl, err := f.DirContainsExcludeFile(ctx, fs, remote)
if err != nil {
return false, err
}
@@ -431,9 +432,9 @@ func (f *Filter) IncludeDirectory(fs fs.Fs) func(string) (bool, error) {
// DirContainsExcludeFile checks if exclude file is present in a
// directroy. If fs is nil, it works properly if ExcludeFile is an
// empty string (for testing).
func (f *Filter) DirContainsExcludeFile(fremote fs.Fs, remote string) (bool, error) {
func (f *Filter) DirContainsExcludeFile(ctx context.Context, fremote fs.Fs, remote string) (bool, error) {
if len(f.Opt.ExcludeFile) > 0 {
exists, err := fs.FileExists(fremote, path.Join(remote, f.Opt.ExcludeFile))
exists, err := fs.FileExists(ctx, fremote, path.Join(remote, f.Opt.ExcludeFile))
if err != nil {
return false, err
}
@@ -470,11 +471,11 @@ func (f *Filter) Include(remote string, size int64, modTime time.Time) bool {
// IncludeObject returns whether this object should be included into
// the sync or not. This is a convenience function to avoid calling
// o.ModTime(), which is an expensive operation.
func (f *Filter) IncludeObject(o fs.Object) bool {
func (f *Filter) IncludeObject(ctx context.Context, o fs.Object) bool {
var modTime time.Time
if !f.ModTimeFrom.IsZero() || !f.ModTimeTo.IsZero() {
modTime = o.ModTime()
modTime = o.ModTime(ctx)
} else {
modTime = time.Unix(0, 0)
}
@@ -534,8 +535,8 @@ func (f *Filter) HaveFilesFrom() bool {
var errFilesFromNotSet = errors.New("--files-from not set so can't use Filter.ListR")
// MakeListR makes function to return all the files set using --files-from
func (f *Filter) MakeListR(NewObject func(remote string) (fs.Object, error)) fs.ListRFn {
return func(dir string, callback fs.ListRCallback) error {
func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Context, remote string) (fs.Object, error)) fs.ListRFn {
return func(ctx context.Context, dir string, callback fs.ListRCallback) error {
if !f.HaveFilesFrom() {
return errFilesFromNotSet
}
@@ -547,7 +548,7 @@ func (f *Filter) MakeListR(NewObject func(remote string) (fs.Object, error)) fs.
g.Go(func() (err error) {
var entries = make(fs.DirEntries, 1)
for remote := range remotes {
entries[0], err = NewObject(remote)
entries[0], err = NewObject(ctx, remote)
if err == fs.ErrorObjectNotFound {
// Skip files that are not found
} else if err != nil {

15
fs/filter/filter_test.go
View File

@@ -1,6 +1,7 @@
package filter
import (
"context"
"fmt"
"io/ioutil"
"os"
@@ -159,7 +160,7 @@ type includeDirTest struct {
func testDirInclude(t *testing.T, f *Filter, tests []includeDirTest) {
for _, test := range tests {
got, err := f.IncludeDirectory(nil)(test.in)
got, err := f.IncludeDirectory(context.Background(), nil)(test.in)
require.NoError(t, err)
assert.Equal(t, test.want, got, test.in)
}
@@ -235,8 +236,8 @@ func TestNewFilterMakeListR(t *testing.T) {
require.NoError(t, err)
// Check error if no files
listR := f.MakeListR(nil)
err = listR("", nil)
listR := f.MakeListR(context.Background(), nil)
err = listR(context.Background(), "", nil)
assert.EqualError(t, err, errFilesFromNotSet.Error())
// Add some files
@@ -256,7 +257,7 @@ func TestNewFilterMakeListR(t *testing.T) {
// NewObject function for MakeListR
newObjects := FilesMap{}
var newObjectMu sync.Mutex
NewObject := func(remote string) (fs.Object, error) {
NewObject := func(ctx context.Context, remote string) (fs.Object, error) {
newObjectMu.Lock()
defer newObjectMu.Unlock()
if remote == "notfound" {
@@ -282,8 +283,8 @@ func TestNewFilterMakeListR(t *testing.T) {
}
// Make the listR and call it
listR = f.MakeListR(NewObject)
err = listR("", listRcallback)
listR = f.MakeListR(context.Background(), NewObject)
err = listR(context.Background(), "", listRcallback)
require.NoError(t, err)
// Check that the correct objects were created and listed
@@ -298,7 +299,7 @@ func TestNewFilterMakeListR(t *testing.T) {
// Now check an error is returned from NewObject
require.NoError(t, f.AddFile("error"))
err = listR("", listRcallback)
err = listR(context.Background(), "", listRcallback)
require.EqualError(t, err, assert.AnError.Error())
}

83
fs/fs.go
View File

@@ -2,6 +2,7 @@
package fs
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -245,11 +246,11 @@ type Fs interface {
//
// This should return ErrDirNotFound if the directory isn't
// found.
List(dir string) (entries DirEntries, err error)
List(ctx context.Context, dir string) (entries DirEntries, err error)
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
NewObject(remote string) (Object, error)
NewObject(ctx context.Context, remote string) (Object, error)
// Put in to the remote path with the modTime given of the given size
//
@@ -260,17 +261,17 @@ type Fs interface {
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
Put(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
Put(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
Mkdir(dir string) error
Mkdir(ctx context.Context, dir string) error
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
Rmdir(dir string) error
Rmdir(ctx context.Context, dir string) error
}
// Info provides a read only interface to information about a filesystem.
@@ -299,20 +300,20 @@ type Object interface {
ObjectInfo
// SetModTime sets the metadata on the object to set the modification date
SetModTime(time.Time) error
SetModTime(ctx context.Context, t time.Time) error
// Open opens the file for read. Call Close() on the returned io.ReadCloser
Open(options ...OpenOption) (io.ReadCloser, error)
Open(ctx context.Context, options ...OpenOption) (io.ReadCloser, error)
// Update in to the object with the modTime given of the given size
//
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
// return an error or update the object properly (rather than e.g. calling panic).
Update(in io.Reader, src ObjectInfo, options ...OpenOption) error
Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error
// Removes this object
Remove() error
Remove(ctx context.Context) error
}
// ObjectInfo provides read only information about an object.
@@ -324,7 +325,7 @@ type ObjectInfo interface {
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
Hash(hash.Type) (string, error)
Hash(ctx context.Context, ty hash.Type) (string, error)
// Storable says whether this object can be stored
Storable() bool
@@ -342,7 +343,7 @@ type DirEntry interface {
// ModTime returns the modification date of the file
// It should return a best guess if one isn't available
ModTime() time.Time
ModTime(context.Context) time.Time
// Size returns the size of the file
Size() int64
@@ -365,7 +366,7 @@ type Directory interface {
type MimeTyper interface {
// MimeType returns the content type of the Object if
// known, or "" if not
MimeType() string
MimeType(ctx context.Context) string
}
// IDer is an optional interface for Object
@@ -430,7 +431,7 @@ func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) {
type ListRCallback func(entries DirEntries) error
// ListRFn is defines the call used to recursively list a directory
type ListRFn func(dir string, callback ListRCallback) error
type ListRFn func(ctx context.Context, dir string, callback ListRCallback) error
// NewUsageValue makes a valid value
func NewUsageValue(value int64) *int64 {
@@ -476,7 +477,7 @@ type Features struct {
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
Purge func() error
Purge func(ctx context.Context) error
// Copy src to this remote using server side copy operations.
//
@@ -487,7 +488,7 @@ type Features struct {
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
Copy func(src Object, remote string) (Object, error)
Copy func(ctx context.Context, src Object, remote string) (Object, error)
// Move src to this remote using server side move operations.
//
@@ -498,7 +499,7 @@ type Features struct {
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
Move func(src Object, remote string) (Object, error)
Move func(ctx context.Context, src Object, remote string) (Object, error)
// DirMove moves src, srcRemote to this remote at dstRemote
// using server side move operations.
@@ -508,12 +509,12 @@ type Features struct {
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
DirMove func(src Fs, srcRemote, dstRemote string) error
DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error
// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
ChangeNotify func(func(string, EntryType), <-chan time.Duration)
ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration)
// UnWrap returns the Fs that this Fs is wrapping
UnWrap func() Fs
@@ -529,7 +530,7 @@ type Features struct {
DirCacheFlush func()
// PublicLink generates a public link to the remote path (usually readable by anyone)
PublicLink func(remote string) (string, error)
PublicLink func(ctx context.Context, remote string) (string, error)
// Put in to the remote path with the modTime given of the given size
//
@@ -539,24 +540,24 @@ type Features struct {
//
// May create duplicates or return errors if src already
// exists.
PutUnchecked func(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
PutStream func(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
// MergeDirs merges the contents of all the directories passed
// in into the first one and rmdirs the other directories.
MergeDirs func([]Directory) error
MergeDirs func(ctx context.Context, dirs []Directory) error
// CleanUp the trash in the Fs
//
// Implement this if you have a way of emptying the trash or
// otherwise cleaning up old versions of files.
CleanUp func() error
CleanUp func(ctx context.Context) error
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
@@ -577,14 +578,14 @@ type Features struct {
ListR ListRFn
// About gets quota information from the Fs
About func() (*Usage, error)
About func(ctx context.Context) (*Usage, error)
// OpenWriterAt opens with a handle for random access writes
//
// Pass in the remote desired and the size if known.
//
// It truncates any existing object
OpenWriterAt func(remote string, size int64) (WriterAtCloser, error)
OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
}
// Disable nil's out the named feature. If it isn't found then it
@@ -803,7 +804,7 @@ type Purger interface {
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
Purge() error
Purge(ctx context.Context) error
}
// Copier is an optional interface for Fs
@@ -817,7 +818,7 @@ type Copier interface {
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
Copy(src Object, remote string) (Object, error)
Copy(ctx context.Context, src Object, remote string) (Object, error)
}
// Mover is an optional interface for Fs
@@ -831,7 +832,7 @@ type Mover interface {
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
Move(src Object, remote string) (Object, error)
Move(ctx context.Context, src Object, remote string) (Object, error)
}
// DirMover is an optional interface for Fs
@@ -844,7 +845,7 @@ type DirMover interface {
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
DirMove(src Fs, srcRemote, dstRemote string) error
DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error
}
// ChangeNotifier is an optional interface for Fs
@@ -858,7 +859,7 @@ type ChangeNotifier interface {
// The ChangeNotify implementation must empty the channel
// regularly. When the channel gets closed, the implementation
// should stop polling and release resources.
ChangeNotify(func(string, EntryType), <-chan time.Duration)
ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration)
}
// UnWrapper is an optional interfaces for Fs
@@ -892,7 +893,7 @@ type PutUncheckeder interface {
//
// May create duplicates or return errors if src already
// exists.
PutUnchecked(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}
// PutStreamer is an optional interface for Fs
@@ -902,20 +903,20 @@ type PutStreamer interface {
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
PutStream(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}
// PublicLinker is an optional interface for Fs
type PublicLinker interface {
// PublicLink generates a public link to the remote path (usually readable by anyone)
PublicLink(remote string) (string, error)
PublicLink(ctx context.Context, remote string) (string, error)
}
// MergeDirser is an option interface for Fs
type MergeDirser interface {
// MergeDirs merges the contents of all the directories passed
// in into the first one and rmdirs the other directories.
MergeDirs([]Directory) error
MergeDirs(ctx context.Context, dirs []Directory) error
}
// CleanUpper is an optional interfaces for Fs
@@ -924,7 +925,7 @@ type CleanUpper interface {
//
// Implement this if you have a way of emptying the trash or
// otherwise cleaning up old versions of files.
CleanUp() error
CleanUp(ctx context.Context) error
}
// ListRer is an optional interfaces for Fs
@@ -945,7 +946,7 @@ type ListRer interface {
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
ListR(dir string, callback ListRCallback) error
ListR(ctx context.Context, dir string, callback ListRCallback) error
}
// RangeSeeker is the interface that wraps the RangeSeek method.
@@ -958,13 +959,13 @@ type RangeSeeker interface {
// limiting the total length to limit.
//
// RangeSeek with a limit of < 0 is equivalent to a regular Seek.
RangeSeek(offset int64, whence int, length int64) (int64, error)
RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error)
}
// Abouter is an optional interface for Fs
type Abouter interface {
// About gets quota information from the Fs
About() (*Usage, error)
About(ctx context.Context) (*Usage, error)
}
// OpenWriterAter is an optional interface for Fs
@@ -974,7 +975,7 @@ type OpenWriterAter interface {
// Pass in the remote desired and the size if known.
//
// It truncates any existing object
OpenWriterAt(remote string, size int64) (WriterAtCloser, error)
OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
}
// ObjectsChan is a channel of Objects
@@ -1195,8 +1196,8 @@ func CheckClose(c io.Closer, err *error) {
// FileExists returns true if a file remote exists.
// If remote is a directory, FileExists returns false.
func FileExists(fs Fs, remote string) (bool, error) {
_, err := fs.NewObject(remote)
func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
_, err := fs.NewObject(ctx, remote)
if err != nil {
if err == ErrorObjectNotFound || err == ErrorNotAFile || err == ErrorPermissionDenied {
return false, nil

7
fs/fs_test.go
View File

@@ -1,6 +1,7 @@
package fs
import (
"context"
"strings"
"sync"
"testing"
@@ -17,7 +18,7 @@ import (
func TestFeaturesDisable(t *testing.T) {
ft := new(Features)
ft.Copy = func(src Object, remote string) (Object, error) {
ft.Copy = func(ctx context.Context, src Object, remote string) (Object, error) {
return nil, nil
}
ft.CaseInsensitive = true
@@ -44,7 +45,7 @@ func TestFeaturesList(t *testing.T) {
func TestFeaturesEnabled(t *testing.T) {
ft := new(Features)
ft.CaseInsensitive = true
ft.Purge = func() error { return nil }
ft.Purge = func(ctx context.Context) error { return nil }
enabled := ft.Enabled()
flag, ok := enabled["CaseInsensitive"]
@@ -68,7 +69,7 @@ func TestFeaturesEnabled(t *testing.T) {
func TestFeaturesDisableList(t *testing.T) {
ft := new(Features)
ft.Copy = func(src Object, remote string) (Object, error) {
ft.Copy = func(ctx context.Context, src Object, remote string) (Object, error) {
return nil, nil
}
ft.CaseInsensitive = true

13
fs/list/list.go
View File

@@ -2,6 +2,7 @@
package list
import (
"context"
"sort"
"strings"
@@ -18,9 +19,9 @@ import (
// files and directories passing the filter will be added.
//
// Files will be returned in sorted order
func DirSorted(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
// Get unfiltered entries from the fs
entries, err = f.List(dir)
entries, err = f.List(ctx, dir)
if err != nil {
return nil, err
}
@@ -31,12 +32,12 @@ func DirSorted(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err
fs.Debugf(dir, "Excluded")
return nil, nil
}
return filterAndSortDir(entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(f))
return filterAndSortDir(ctx, entries, includeAll, dir, filter.Active.IncludeObject, filter.Active.IncludeDirectory(ctx, f))
}
// filter (if required) and check the entries, then sort them
func filterAndSortDir(entries fs.DirEntries, includeAll bool, dir string,
IncludeObject func(o fs.Object) bool,
func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
IncludeObject func(ctx context.Context, o fs.Object) bool,
IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
newEntries = entries[:0] // in place filter
prefix := ""
@@ -49,7 +50,7 @@ func filterAndSortDir(entries fs.DirEntries, includeAll bool, dir string,
switch x := entry.(type) {
case fs.Object:
// Make sure we don't delete excluded files if not required
if !includeAll && !IncludeObject(x) {
if !includeAll && !IncludeObject(ctx, x) {
ok = false
fs.Debugf(x, "Excluded")
}

21
fs/list/list_test.go
View File

@@ -1,6 +1,7 @@
package list
import (
"context"
"testing"
"time"
@@ -24,21 +25,21 @@ func TestFilterAndSortIncludeAll(t *testing.T) {
dd := mockdir.New("d")
oD := mockobject.Object("D")
entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
includeObject := func(o fs.Object) bool {
includeObject := func(ctx context.Context, o fs.Object) bool {
return o != oB
}
includeDirectory := func(remote string) (bool, error) {
return remote != "c", nil
}
// no filter
newEntries, err := filterAndSortDir(entries, true, "", includeObject, includeDirectory)
newEntries, err := filterAndSortDir(context.Background(), entries, true, "", includeObject, includeDirectory)
require.NoError(t, err)
assert.Equal(t,
newEntries,
fs.DirEntries{oA, oB, oC, oD, da, db, dc, dd},
)
// filter
newEntries, err = filterAndSortDir(entries, false, "", includeObject, includeDirectory)
newEntries, err = filterAndSortDir(context.Background(), entries, false, "", includeObject, includeDirectory)
require.NoError(t, err)
assert.Equal(t,
newEntries,
@@ -57,7 +58,7 @@ func TestFilterAndSortCheckDir(t *testing.T) {
dd := mockdir.New("dir/d")
oD := mockobject.Object("dir/D")
entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
newEntries, err := filterAndSortDir(entries, true, "dir", nil, nil)
newEntries, err := filterAndSortDir(context.Background(), entries, true, "dir", nil, nil)
require.NoError(t, err)
assert.Equal(t,
newEntries,
@@ -76,7 +77,7 @@ func TestFilterAndSortCheckDirRoot(t *testing.T) {
dd := mockdir.New("d")
oD := mockobject.Object("D")
entries := fs.DirEntries{da, oA, db, oB, dc, oC, dd, oD}
newEntries, err := filterAndSortDir(entries, true, "", nil, nil)
newEntries, err := filterAndSortDir(context.Background(), entries, true, "", nil, nil)
require.NoError(t, err)
assert.Equal(t,
newEntries,
@@ -86,10 +87,10 @@ func TestFilterAndSortCheckDirRoot(t *testing.T) {
type unknownDirEntry string
func (o unknownDirEntry) String() string { return string(o) }
func (o unknownDirEntry) Remote() string { return string(o) }
func (o unknownDirEntry) ModTime() (t time.Time) { return t }
func (o unknownDirEntry) Size() int64 { return 0 }
func (o unknownDirEntry) String() string { return string(o) }
func (o unknownDirEntry) Remote() string { return string(o) }
func (o unknownDirEntry) ModTime(ctx context.Context) (t time.Time) { return t }
func (o unknownDirEntry) Size() int64 { return 0 }
func TestFilterAndSortUnknown(t *testing.T) {
// Check that an unknown entry produces an error
@@ -98,7 +99,7 @@ func TestFilterAndSortUnknown(t *testing.T) {
ub := unknownDirEntry("b")
oB := mockobject.Object("B/sub")
entries := fs.DirEntries{da, oA, ub, oB}
newEntries, err := filterAndSortDir(entries, true, "", nil, nil)
newEntries, err := filterAndSortDir(context.Background(), entries, true, "", nil, nil)
assert.Error(t, err, "error")
assert.Nil(t, newEntries)
}

10
fs/march/march.go
View File

@@ -40,7 +40,7 @@ type Marcher interface {
// DstOnly is called for a DirEntry found only in the destination
DstOnly(dst fs.DirEntry) (recurse bool)
// Match is called for a DirEntry found both in the source and destination
Match(dst, src fs.DirEntry) (recurse bool)
Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool)
}
// init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
@@ -70,7 +70,7 @@ type listDirFn func(dir string) (entries fs.DirEntries, err error)
func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
if (!fs.Config.UseListR || f.Features().ListR == nil) && !filter.Active.HaveFilesFrom() {
return func(dir string) (entries fs.DirEntries, err error) {
return list.DirSorted(f, includeAll, dir)
return list.DirSorted(m.Ctx, f, includeAll, dir)
}
}
var (
@@ -83,7 +83,7 @@ func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
mu.Lock()
defer mu.Unlock()
if !started {
dirs, dirsErr = walk.NewDirTree(f, m.Dir, includeAll, fs.Config.MaxDepth)
dirs, dirsErr = walk.NewDirTree(m.Ctx, f, m.Dir, includeAll, fs.Config.MaxDepth)
started = true
}
if dirsErr != nil {
@@ -383,7 +383,7 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
for _, src := range srcList {
if srcObj, ok := src.(fs.Object); ok {
leaf := path.Base(srcObj.Remote())
dstObj, err := m.Fdst.NewObject(path.Join(job.dstRemote, leaf))
dstObj, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))
if err == nil {
dstList = append(dstList, dstObj)
}
@@ -424,7 +424,7 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) {
if m.aborting() {
return nil
}
recurse := m.Callback.Match(match.dst, match.src)
recurse := m.Callback.Match(m.Ctx, match.dst, match.src)
if recurse && job.srcDepth > 0 && job.dstDepth > 0 {
jobs = append(jobs, listDirJob{
srcRemote: match.src.Remote(),

9
fs/mimetype.go
View File

@@ -1,6 +1,7 @@
package fs
import (
"context"
"mime"
"path"
"strings"
@@ -17,10 +18,10 @@ func MimeTypeFromName(remote string) (mimeType string) {
// MimeType returns the MimeType from the object, either by calling
// the MimeTyper interface or using MimeTypeFromName
func MimeType(o ObjectInfo) (mimeType string) {
func MimeType(ctx context.Context, o ObjectInfo) (mimeType string) {
// Read the MimeType from the optional interface if available
if do, ok := o.(MimeTyper); ok {
mimeType = do.MimeType()
mimeType = do.MimeType(ctx)
// Debugf(o, "Read MimeType as %q", mimeType)
if mimeType != "" {
return mimeType
@@ -33,10 +34,10 @@ func MimeType(o ObjectInfo) (mimeType string) {
//
// It returns "inode/directory" for directories, or uses
// MimeType(Object)
func MimeTypeDirEntry(item DirEntry) string {
func MimeTypeDirEntry(ctx context.Context, item DirEntry) string {
switch x := item.(type) {
case Object:
return MimeType(x)
return MimeType(ctx, x)
case Directory:
return "inode/directory"
}

43
fs/object/object.go
View File

@@ -3,6 +3,7 @@ package object
import (
"bytes"
"context"
"errors"
"io"
"io/ioutil"
@@ -43,13 +44,13 @@ type staticObjectInfo struct {
fs fs.Info
}
func (i *staticObjectInfo) Fs() fs.Info { return i.fs }
func (i *staticObjectInfo) Remote() string { return i.remote }
func (i *staticObjectInfo) String() string { return i.remote }
func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
func (i *staticObjectInfo) Size() int64 { return i.size }
func (i *staticObjectInfo) Storable() bool { return i.storable }
func (i *staticObjectInfo) Hash(h hash.Type) (string, error) {
func (i *staticObjectInfo) Fs() fs.Info { return i.fs }
func (i *staticObjectInfo) Remote() string { return i.remote }
func (i *staticObjectInfo) String() string { return i.remote }
func (i *staticObjectInfo) ModTime(ctx context.Context) time.Time { return i.modTime }
func (i *staticObjectInfo) Size() int64 { return i.size }
func (i *staticObjectInfo) Storable() bool { return i.storable }
func (i *staticObjectInfo) Hash(ctx context.Context, h hash.Type) (string, error) {
if len(i.hashes) == 0 {
return "", hash.ErrUnsupported
}
@@ -92,13 +93,13 @@ func (memoryFs) Features() *fs.Features { return &fs.Features{} }
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (memoryFs) List(dir string) (entries fs.DirEntries, err error) {
func (memoryFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
return nil, nil
}
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
func (memoryFs) NewObject(remote string) (fs.Object, error) {
func (memoryFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return nil, fs.ErrorObjectNotFound
}
@@ -107,22 +108,22 @@ func (memoryFs) NewObject(remote string) (fs.Object, error) {
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (memoryFs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o := NewMemoryObject(src.Remote(), src.ModTime(), nil)
return o, o.Update(in, src, options...)
func (memoryFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o := NewMemoryObject(src.Remote(), src.ModTime(ctx), nil)
return o, o.Update(ctx, in, src, options...)
}
// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
func (memoryFs) Mkdir(dir string) error {
func (memoryFs) Mkdir(ctx context.Context, dir string) error {
return errors.New("memoryFs: can't make directory")
}
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (memoryFs) Rmdir(dir string) error {
func (memoryFs) Rmdir(ctx context.Context, dir string) error {
return fs.ErrorDirNotFound
}
@@ -165,7 +166,7 @@ func (o *MemoryObject) String() string {
}
// ModTime returns the modification date of the file
func (o *MemoryObject) ModTime() time.Time {
func (o *MemoryObject) ModTime(ctx context.Context) time.Time {
return o.modTime
}
@@ -180,7 +181,7 @@ func (o *MemoryObject) Storable() bool {
}
// Hash returns the requested hash of the contents
func (o *MemoryObject) Hash(h hash.Type) (string, error) {
func (o *MemoryObject) Hash(ctx context.Context, h hash.Type) (string, error) {
hash, err := hash.NewMultiHasherTypes(hash.Set(h))
if err != nil {
return "", err
@@ -193,13 +194,13 @@ func (o *MemoryObject) Hash(h hash.Type) (string, error) {
}
// SetModTime sets the metadata on the object to set the modification date
func (o *MemoryObject) SetModTime(modTime time.Time) error {
func (o *MemoryObject) SetModTime(ctx context.Context, modTime time.Time) error {
o.modTime = modTime
return nil
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *MemoryObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
func (o *MemoryObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
content := o.content
for _, option := range options {
switch x := option.(type) {
@@ -219,7 +220,7 @@ func (o *MemoryObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
// Update in to the object with the modTime given of the given size
//
// This re-uses the internal buffer if at all possible.
func (o *MemoryObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
func (o *MemoryObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
size := src.Size()
if size == 0 {
o.content = nil
@@ -229,11 +230,11 @@ func (o *MemoryObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.Ope
o.content = o.content[:size]
_, err = io.ReadFull(in, o.content)
}
o.modTime = src.ModTime()
o.modTime = src.ModTime(ctx)
return err
}
// Remove this object
func (o *MemoryObject) Remove() error {
func (o *MemoryObject) Remove(ctx context.Context) error {
return errors.New("memoryObject.Remove not supported")
}

51
fs/object/object_test.go
View File

@@ -2,6 +2,7 @@ package object_test
import (
"bytes"
"context"
"io"
"io/ioutil"
"testing"
@@ -23,26 +24,26 @@ func TestStaticObject(t *testing.T) {
assert.Equal(t, object.MemoryFs, o.Fs())
assert.Equal(t, remote, o.Remote())
assert.Equal(t, remote, o.String())
assert.Equal(t, now, o.ModTime())
assert.Equal(t, now, o.ModTime(context.Background()))
assert.Equal(t, size, o.Size())
assert.Equal(t, true, o.Storable())
Hash, err := o.Hash(hash.MD5)
Hash, err := o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "", Hash)
o = object.NewStaticObjectInfo(remote, now, size, true, nil, nil)
_, err = o.Hash(hash.MD5)
_, err = o.Hash(context.Background(), hash.MD5)
assert.Equal(t, hash.ErrUnsupported, err)
hs := map[hash.Type]string{
hash.MD5: "potato",
}
o = object.NewStaticObjectInfo(remote, now, size, true, hs, nil)
Hash, err = o.Hash(hash.MD5)
Hash, err = o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "potato", Hash)
_, err = o.Hash(hash.SHA1)
_, err = o.Hash(context.Background(), hash.SHA1)
assert.Equal(t, hash.ErrUnsupported, err)
}
@@ -55,27 +56,27 @@ func TestMemoryFs(t *testing.T) {
assert.Equal(t, hash.Supported, f.Hashes())
assert.Equal(t, &fs.Features{}, f.Features())
entries, err := f.List("")
entries, err := f.List(context.Background(), "")
assert.NoError(t, err)
assert.Nil(t, entries)
o, err := f.NewObject("obj")
o, err := f.NewObject(context.Background(), "obj")
assert.Equal(t, fs.ErrorObjectNotFound, err)
assert.Nil(t, o)
buf := bytes.NewBufferString("potato")
now := time.Now()
src := object.NewStaticObjectInfo("remote", now, int64(buf.Len()), true, nil, nil)
o, err = f.Put(buf, src)
o, err = f.Put(context.Background(), buf, src)
assert.NoError(t, err)
hash, err := o.Hash(hash.SHA1)
hash, err := o.Hash(context.Background(), hash.SHA1)
assert.NoError(t, err)
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", hash)
err = f.Mkdir("dir")
err = f.Mkdir(context.Background(), "dir")
assert.Error(t, err)
err = f.Rmdir("dir")
err = f.Rmdir(context.Background(), "dir")
assert.Equal(t, fs.ErrorDirNotFound, err)
}
@@ -91,22 +92,22 @@ func TestMemoryObject(t *testing.T) {
assert.Equal(t, object.MemoryFs, o.Fs())
assert.Equal(t, remote, o.Remote())
assert.Equal(t, remote, o.String())
assert.Equal(t, now, o.ModTime())
assert.Equal(t, now, o.ModTime(context.Background()))
assert.Equal(t, int64(len(content)), o.Size())
assert.Equal(t, true, o.Storable())
Hash, err := o.Hash(hash.MD5)
Hash, err := o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "8ee2027983915ec78acc45027d874316", Hash)
Hash, err = o.Hash(hash.SHA1)
Hash, err = o.Hash(context.Background(), hash.SHA1)
assert.NoError(t, err)
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", Hash)
newNow := now.Add(time.Minute)
err = o.SetModTime(newNow)
err = o.SetModTime(context.Background(), newNow)
assert.NoError(t, err)
assert.Equal(t, newNow, o.ModTime())
assert.Equal(t, newNow, o.ModTime(context.Background()))
checkOpen := func(rc io.ReadCloser, expected string) {
actual, err := ioutil.ReadAll(rc)
@@ -117,18 +118,18 @@ func TestMemoryObject(t *testing.T) {
}
checkContent := func(o fs.Object, expected string) {
rc, err := o.Open()
rc, err := o.Open(context.Background())
assert.NoError(t, err)
checkOpen(rc, expected)
}
checkContent(o, string(content))
rc, err := o.Open(&fs.RangeOption{Start: 1, End: 3})
rc, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 3})
assert.NoError(t, err)
checkOpen(rc, "ot")
rc, err = o.Open(&fs.SeekOption{Offset: 3})
rc, err = o.Open(context.Background(), &fs.SeekOption{Offset: 3})
assert.NoError(t, err)
checkOpen(rc, "ato")
@@ -137,10 +138,10 @@ func TestMemoryObject(t *testing.T) {
newContent := bytes.NewBufferString("Rutabaga")
assert.True(t, newContent.Len() < cap(content)) // fits within cap(content)
src := object.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, "Rutabaga")
assert.Equal(t, newNow, o.ModTime())
assert.Equal(t, newNow, o.ModTime(context.Background()))
assert.Equal(t, "Rutaba", string(content)) // check we re-used the buffer
// not within the buffer
@@ -149,7 +150,7 @@ func TestMemoryObject(t *testing.T) {
newContent = bytes.NewBufferString(newStr)
assert.True(t, newContent.Len() > cap(content)) // does not fit within cap(content)
src = object.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
assert.Equal(t, "Rutaba", string(content)) // check we didn't re-use the buffer
@@ -158,7 +159,7 @@ func TestMemoryObject(t *testing.T) {
newStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
newContent = bytes.NewBufferString(newStr)
src = object.NewStaticObjectInfo(remote, newNow, -1, true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
@@ -166,10 +167,10 @@ func TestMemoryObject(t *testing.T) {
newStr = ""
newContent = bytes.NewBufferString(newStr)
src = object.NewStaticObjectInfo(remote, newNow, 0, true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
err = o.Remove()
err = o.Remove(context.Background())
assert.Error(t, err)
}

61
fs/operations/dedupe.go
View File

@@ -3,6 +3,7 @@
package operations
import (
"context"
"fmt"
"log"
"path"
@@ -18,7 +19,7 @@ import (
)
// dedupeRename renames the objs slice to different names
func dedupeRename(f fs.Fs, remote string, objs []fs.Object) {
func dedupeRename(ctx context.Context, f fs.Fs, remote string, objs []fs.Object) {
doMove := f.Features().Move
if doMove == nil {
log.Fatalf("Fs %v doesn't support Move", f)
@@ -30,7 +31,7 @@ outer:
for i, o := range objs {
suffix := 1
newName := fmt.Sprintf("%s-%d%s", base, i+suffix, ext)
_, err := f.NewObject(newName)
_, err := f.NewObject(ctx, newName)
for ; err != fs.ErrorObjectNotFound; suffix++ {
if err != nil {
fs.CountError(err)
@@ -42,10 +43,10 @@ outer:
continue outer
}
newName = fmt.Sprintf("%s-%d%s", base, i+suffix, ext)
_, err = f.NewObject(newName)
_, err = f.NewObject(ctx, newName)
}
if !fs.Config.DryRun {
newObj, err := doMove(o, newName)
newObj, err := doMove(ctx, o, newName)
if err != nil {
fs.CountError(err)
fs.Errorf(o, "Failed to rename: %v", err)
@@ -59,22 +60,22 @@ outer:
}
// dedupeDeleteAllButOne deletes all but the one in keep
func dedupeDeleteAllButOne(keep int, remote string, objs []fs.Object) {
func dedupeDeleteAllButOne(ctx context.Context, keep int, remote string, objs []fs.Object) {
for i, o := range objs {
if i == keep {
continue
}
_ = DeleteFile(o)
_ = DeleteFile(ctx, o)
}
fs.Logf(remote, "Deleted %d extra copies", len(objs)-1)
}
// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
func dedupeDeleteIdentical(ht hash.Type, remote string, objs []fs.Object) (remainingObjs []fs.Object) {
func dedupeDeleteIdentical(ctx context.Context, ht hash.Type, remote string, objs []fs.Object) (remainingObjs []fs.Object) {
// See how many of these duplicates are identical
byHash := make(map[string][]fs.Object, len(objs))
for _, o := range objs {
md5sum, err := o.Hash(ht)
md5sum, err := o.Hash(ctx, ht)
if err != nil || md5sum == "" {
remainingObjs = append(remainingObjs, o)
} else {
@@ -87,7 +88,7 @@ func dedupeDeleteIdentical(ht hash.Type, remote string, objs []fs.Object) (remai
if len(hashObjs) > 1 {
fs.Logf(remote, "Deleting %d/%d identical duplicates (%v %q)", len(hashObjs)-1, len(hashObjs), ht, md5sum)
for _, o := range hashObjs[1:] {
_ = DeleteFile(o)
_ = DeleteFile(ctx, o)
}
}
remainingObjs = append(remainingObjs, hashObjs[0])
@@ -97,22 +98,22 @@ func dedupeDeleteIdentical(ht hash.Type, remote string, objs []fs.Object) (remai
}
// dedupeInteractive interactively dedupes the slice of objects
func dedupeInteractive(f fs.Fs, ht hash.Type, remote string, objs []fs.Object) {
func dedupeInteractive(ctx context.Context, f fs.Fs, ht hash.Type, remote string, objs []fs.Object) {
fmt.Printf("%s: %d duplicates remain\n", remote, len(objs))
for i, o := range objs {
md5sum, err := o.Hash(ht)
md5sum, err := o.Hash(ctx, ht)
if err != nil {
md5sum = err.Error()
}
fmt.Printf(" %d: %12d bytes, %s, %v %32s\n", i+1, o.Size(), o.ModTime().Local().Format("2006-01-02 15:04:05.000000000"), ht, md5sum)
fmt.Printf(" %d: %12d bytes, %s, %v %32s\n", i+1, o.Size(), o.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), ht, md5sum)
}
switch config.Command([]string{"sSkip and do nothing", "kKeep just one (choose which in next step)", "rRename all to be different (by changing file.jpg to file-1.jpg)"}) {
case 's':
case 'k':
keep := config.ChooseNumber("Enter the number of the file to keep", 1, len(objs))
dedupeDeleteAllButOne(keep-1, remote, objs)
dedupeDeleteAllButOne(ctx, keep-1, remote, objs)
case 'r':
dedupeRename(f, remote, objs)
dedupeRename(ctx, f, remote, objs)
}
}
@@ -121,7 +122,7 @@ type objectsSortedByModTime []fs.Object
func (objs objectsSortedByModTime) Len() int { return len(objs) }
func (objs objectsSortedByModTime) Swap(i, j int) { objs[i], objs[j] = objs[j], objs[i] }
func (objs objectsSortedByModTime) Less(i, j int) bool {
return objs[i].ModTime().Before(objs[j].ModTime())
return objs[i].ModTime(context.TODO()).Before(objs[j].ModTime(context.TODO()))
}
// DeduplicateMode is how the dedupe command chooses what to do
@@ -190,9 +191,9 @@ func (x *DeduplicateMode) Type() string {
var _ pflag.Value = (*DeduplicateMode)(nil)
// dedupeFindDuplicateDirs scans f for duplicate directories
func dedupeFindDuplicateDirs(f fs.Fs) ([][]fs.Directory, error) {
func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) ([][]fs.Directory, error) {
dirs := map[string][]fs.Directory{}
err := walk.ListR(f, "", true, fs.Config.MaxDepth, walk.ListDirs, func(entries fs.DirEntries) error {
err := walk.ListR(ctx, f, "", true, fs.Config.MaxDepth, walk.ListDirs, func(entries fs.DirEntries) error {
entries.ForDir(func(d fs.Directory) {
dirs[d.Remote()] = append(dirs[d.Remote()], d)
})
@@ -211,7 +212,7 @@ func dedupeFindDuplicateDirs(f fs.Fs) ([][]fs.Directory, error) {
}
// dedupeMergeDuplicateDirs merges all the duplicate directories found
func dedupeMergeDuplicateDirs(f fs.Fs, duplicateDirs [][]fs.Directory) error {
func dedupeMergeDuplicateDirs(ctx context.Context, f fs.Fs, duplicateDirs [][]fs.Directory) error {
mergeDirs := f.Features().MergeDirs
if mergeDirs == nil {
return errors.Errorf("%v: can't merge directories", f)
@@ -223,7 +224,7 @@ func dedupeMergeDuplicateDirs(f fs.Fs, duplicateDirs [][]fs.Directory) error {
for _, dirs := range duplicateDirs {
if !fs.Config.DryRun {
fs.Infof(dirs[0], "Merging contents of duplicate directories")
err := mergeDirs(dirs)
err := mergeDirs(ctx, dirs)
if err != nil {
return errors.Wrap(err, "merge duplicate dirs")
}
@@ -238,20 +239,20 @@ func dedupeMergeDuplicateDirs(f fs.Fs, duplicateDirs [][]fs.Directory) error {
// Deduplicate interactively finds duplicate files and offers to
// delete all but one or rename them to be different. Only useful with
// Google Drive which can have duplicate file names.
func Deduplicate(f fs.Fs, mode DeduplicateMode) error {
func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode) error {
fs.Infof(f, "Looking for duplicates using %v mode.", mode)
// Find duplicate directories first and fix them - repeat
// until all fixed
for {
duplicateDirs, err := dedupeFindDuplicateDirs(f)
duplicateDirs, err := dedupeFindDuplicateDirs(ctx, f)
if err != nil {
return err
}
if len(duplicateDirs) == 0 {
break
}
err = dedupeMergeDuplicateDirs(f, duplicateDirs)
err = dedupeMergeDuplicateDirs(ctx, f, duplicateDirs)
if err != nil {
return err
}
@@ -265,7 +266,7 @@ func Deduplicate(f fs.Fs, mode DeduplicateMode) error {
// Now find duplicate files
files := map[string][]fs.Object{}
err := walk.ListR(f, "", true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
err := walk.ListR(ctx, f, "", true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(o fs.Object) {
remote := o.Remote()
files[remote] = append(files[remote], o)
@@ -279,24 +280,24 @@ func Deduplicate(f fs.Fs, mode DeduplicateMode) error {
for remote, objs := range files {
if len(objs) > 1 {
fs.Logf(remote, "Found %d duplicates - deleting identical copies", len(objs))
objs = dedupeDeleteIdentical(ht, remote, objs)
objs = dedupeDeleteIdentical(ctx, ht, remote, objs)
if len(objs) <= 1 {
fs.Logf(remote, "All duplicates removed")
continue
}
switch mode {
case DeduplicateInteractive:
dedupeInteractive(f, ht, remote, objs)
dedupeInteractive(ctx, f, ht, remote, objs)
case DeduplicateFirst:
dedupeDeleteAllButOne(0, remote, objs)
dedupeDeleteAllButOne(ctx, 0, remote, objs)
case DeduplicateNewest:
sort.Sort(objectsSortedByModTime(objs)) // sort oldest first
dedupeDeleteAllButOne(len(objs)-1, remote, objs)
dedupeDeleteAllButOne(ctx, len(objs)-1, remote, objs)
case DeduplicateOldest:
sort.Sort(objectsSortedByModTime(objs)) // sort oldest first
dedupeDeleteAllButOne(0, remote, objs)
dedupeDeleteAllButOne(ctx, 0, remote, objs)
case DeduplicateRename:
dedupeRename(f, remote, objs)
dedupeRename(ctx, f, remote, objs)
case DeduplicateLargest:
largest, largestIndex := int64(-1), -1
for i, obj := range objs {
@@ -306,7 +307,7 @@ func Deduplicate(f fs.Fs, mode DeduplicateMode) error {
}
}
if largestIndex > -1 {
dedupeDeleteAllButOne(largestIndex, remote, objs)
dedupeDeleteAllButOne(ctx, largestIndex, remote, objs)
}
case DeduplicateSkip:
// skip

75
fs/operations/dedupe_test.go
View File

@@ -1,6 +1,7 @@
package operations_test
import (
"context"
"testing"
"time"
@@ -37,12 +38,12 @@ func TestDeduplicateInteractive(t *testing.T) {
skipIfCantDedupe(t, r.Fremote)
skipIfNoHash(t, r.Fremote)
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject("one", "This is one", t1)
file3 := r.WriteUncheckedObject("one", "This is one", t1)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
r.CheckWithDuplicates(t, file1, file2, file3)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateInteractive)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file1)
@@ -54,17 +55,17 @@ func TestDeduplicateSkip(t *testing.T) {
skipIfCantDedupe(t, r.Fremote)
haveHash := r.Fremote.Hashes().GetOne() != hash.None
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
files := []fstest.Item{file1}
if haveHash {
file2 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
files = append(files, file2)
}
file3 := r.WriteUncheckedObject("one", "This is another one", t1)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
files = append(files, file3)
r.CheckWithDuplicates(t, files...)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateSkip)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip)
require.NoError(t, err)
r.CheckWithDuplicates(t, file1, file3)
@@ -75,18 +76,18 @@ func TestDeduplicateFirst(t *testing.T) {
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject("one", "This is one A", t1)
file3 := r.WriteUncheckedObject("one", "This is one BB", t1)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1)
r.CheckWithDuplicates(t, file1, file2, file3)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateFirst)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst)
require.NoError(t, err)
// list until we get one object
var objects, size int64
for try := 1; try <= *fstest.ListRetries; try++ {
objects, size, err = operations.Count(r.Fremote)
objects, size, err = operations.Count(context.Background(), r.Fremote)
require.NoError(t, err)
if objects == 1 {
break
@@ -104,12 +105,12 @@ func TestDeduplicateNewest(t *testing.T) {
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject("one", "This is one too", t2)
file3 := r.WriteUncheckedObject("one", "This is another one", t3)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
r.CheckWithDuplicates(t, file1, file2, file3)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateNewest)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file3)
@@ -120,12 +121,12 @@ func TestDeduplicateOldest(t *testing.T) {
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject("one", "This is one too", t2)
file3 := r.WriteUncheckedObject("one", "This is another one", t3)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
r.CheckWithDuplicates(t, file1, file2, file3)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateOldest)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file1)
@@ -136,12 +137,12 @@ func TestDeduplicateLargest(t *testing.T) {
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
file1 := r.WriteUncheckedObject("one", "This is one", t1)
file2 := r.WriteUncheckedObject("one", "This is one too", t2)
file3 := r.WriteUncheckedObject("one", "This is another one", t3)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
r.CheckWithDuplicates(t, file1, file2, file3)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateLargest)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file3)
@@ -152,16 +153,16 @@ func TestDeduplicateRename(t *testing.T) {
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
file1 := r.WriteUncheckedObject("one.txt", "This is one", t1)
file2 := r.WriteUncheckedObject("one.txt", "This is one too", t2)
file3 := r.WriteUncheckedObject("one.txt", "This is another one", t3)
file4 := r.WriteUncheckedObject("one-1.txt", "This is not a duplicate", t1)
file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2)
file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3)
file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1)
r.CheckWithDuplicates(t, file1, file2, file3, file4)
err := operations.Deduplicate(r.Fremote, operations.DeduplicateRename)
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename)
require.NoError(t, err)
require.NoError(t, walk.ListR(r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(o fs.Object) {
remote := o.Remote()
if remote != "one-1.txt" &&
@@ -196,23 +197,23 @@ func TestMergeDirs(t *testing.T) {
t.Skip("Can't merge directories")
}
file1 := r.WriteObject("dupe1/one.txt", "This is one", t1)
file2 := r.WriteObject("dupe2/two.txt", "This is one too", t2)
file3 := r.WriteObject("dupe3/three.txt", "This is another one", t3)
file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1)
file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2)
file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3)
objs, dirs, err := walk.GetAll(r.Fremote, "", true, 1)
objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1)
require.NoError(t, err)
assert.Equal(t, 3, len(dirs))
assert.Equal(t, 0, len(objs))
err = mergeDirs(dirs)
err = mergeDirs(context.Background(), dirs)
require.NoError(t, err)
file2.Path = "dupe1/two.txt"
file3.Path = "dupe1/three.txt"
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
objs, dirs, err = walk.GetAll(r.Fremote, "", true, 1)
objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1)
require.NoError(t, err)
assert.Equal(t, 1, len(dirs))
assert.Equal(t, 0, len(objs))

31
fs/operations/listdirsorted_test.go
View File

@@ -1,6 +1,7 @@
package operations_test
import (
"context"
"testing"
"github.com/ncw/rclone/fs"
@@ -23,13 +24,13 @@ func TestListDirSorted(t *testing.T) {
}()
files := []fstest.Item{
r.WriteObject("a.txt", "hello world", t1),
r.WriteObject("zend.txt", "hello", t1),
r.WriteObject("sub dir/hello world", "hello world", t1),
r.WriteObject("sub dir/hello world2", "hello world", t1),
r.WriteObject("sub dir/ignore dir/.ignore", "", t1),
r.WriteObject("sub dir/ignore dir/should be ignored", "to ignore", t1),
r.WriteObject("sub dir/sub sub dir/hello world3", "hello world", t1),
r.WriteObject(context.Background(), "a.txt", "hello world", t1),
r.WriteObject(context.Background(), "zend.txt", "hello", t1),
r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1),
r.WriteObject(context.Background(), "sub dir/hello world2", "hello world", t1),
r.WriteObject(context.Background(), "sub dir/ignore dir/.ignore", "", t1),
r.WriteObject(context.Background(), "sub dir/ignore dir/should be ignored", "to ignore", t1),
r.WriteObject(context.Background(), "sub dir/sub sub dir/hello world3", "hello world", t1),
}
fstest.CheckItems(t, r.Fremote, files...)
var items fs.DirEntries
@@ -50,20 +51,20 @@ func TestListDirSorted(t *testing.T) {
return name
}
items, err = list.DirSorted(r.Fremote, true, "")
items, err = list.DirSorted(context.Background(), r.Fremote, true, "")
require.NoError(t, err)
require.Len(t, items, 3)
assert.Equal(t, "a.txt", str(0))
assert.Equal(t, "sub dir/", str(1))
assert.Equal(t, "zend.txt", str(2))
items, err = list.DirSorted(r.Fremote, false, "")
items, err = list.DirSorted(context.Background(), r.Fremote, false, "")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/", str(0))
assert.Equal(t, "zend.txt", str(1))
items, err = list.DirSorted(r.Fremote, true, "sub dir")
items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir")
require.NoError(t, err)
require.Len(t, items, 4)
assert.Equal(t, "sub dir/hello world", str(0))
@@ -71,7 +72,7 @@ func TestListDirSorted(t *testing.T) {
assert.Equal(t, "sub dir/ignore dir/", str(2))
assert.Equal(t, "sub dir/sub sub dir/", str(3))
items, err = list.DirSorted(r.Fremote, false, "sub dir")
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/", str(0))
@@ -80,23 +81,23 @@ func TestListDirSorted(t *testing.T) {
// testing ignore file
filter.Active.Opt.ExcludeFile = ".ignore"
items, err = list.DirSorted(r.Fremote, false, "sub dir")
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 1)
assert.Equal(t, "sub dir/sub sub dir/", str(0))
items, err = list.DirSorted(r.Fremote, false, "sub dir/ignore dir")
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 0)
items, err = list.DirSorted(r.Fremote, true, "sub dir/ignore dir")
items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
filter.Active.Opt.ExcludeFile = ""
items, err = list.DirSorted(r.Fremote, false, "sub dir/ignore dir")
items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))

11
fs/operations/lsjson.go
View File

@@ -1,6 +1,7 @@
package operations
import (
"context"
"path"
"time"
@@ -78,7 +79,7 @@ type ListJSONOpt struct {
}
// ListJSON lists fsrc using the options in opt calling callback for each item
func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
var cipher crypt.Cipher
if opt.ShowEncrypted {
fsInfo, _, _, config, err := fs.ConfigFs(fsrc.Name() + ":" + fsrc.Root())
@@ -97,7 +98,7 @@ func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJS
canGetTier := features.GetTier
format := formatForPrecision(fsrc.Precision())
isBucket := features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket based remote listing the root mark directories as buckets
err := walk.ListR(fsrc, remote, false, ConfigMaxDepth(opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
err := walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
for _, entry := range entries {
switch entry.(type) {
case fs.Directory:
@@ -116,10 +117,10 @@ func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJS
Path: entry.Remote(),
Name: path.Base(entry.Remote()),
Size: entry.Size(),
MimeType: fs.MimeTypeDirEntry(entry),
MimeType: fs.MimeTypeDirEntry(ctx, entry),
}
if !opt.NoModTime {
item.ModTime = Timestamp{When: entry.ModTime(), Format: format}
item.ModTime = Timestamp{When: entry.ModTime(ctx), Format: format}
}
if cipher != nil {
switch entry.(type) {
@@ -161,7 +162,7 @@ func ListJSON(fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJS
if opt.ShowHash {
item.Hashes = make(map[string]string)
for _, hashType := range x.Fs().Hashes().Array() {
hash, err := x.Hash(hashType)
hash, err := x.Hash(ctx, hashType)
if err != nil {
fs.Errorf(x, "Failed to read hash: %v", err)
} else if hash != "" {

14
fs/operations/multithread.go
View File

@@ -28,7 +28,7 @@ type multiThreadCopyState struct {
}
// Copy a single stream into place
func (mc *multiThreadCopyState) copyStream(stream int) (err error) {
func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err error) {
defer func() {
if err != nil {
fs.Debugf(mc.src, "multi-thread copy: stream %d/%d failed: %v", stream+1, mc.streams, err)
@@ -45,7 +45,7 @@ func (mc *multiThreadCopyState) copyStream(stream int) (err error) {
fs.Debugf(mc.src, "multi-thread copy: stream %d/%d (%d-%d) size %v starting", stream+1, mc.streams, start, end, fs.SizeSuffix(end-start))
rc, err := newReOpen(mc.src, nil, &fs.RangeOption{Start: start, End: end - 1}, fs.Config.LowLevelRetries)
rc, err := newReOpen(ctx, mc.src, nil, &fs.RangeOption{Start: start, End: end - 1}, fs.Config.LowLevelRetries)
if err != nil {
return errors.Wrap(err, "multpart copy: failed to open source")
}
@@ -110,7 +110,7 @@ func (mc *multiThreadCopyState) calculateChunks() {
}
// Copy src to (f, remote) using streams download threads and the OpenWriterAt feature
func multiThreadCopy(f fs.Fs, remote string, src fs.Object, streams int) (newDst fs.Object, err error) {
func multiThreadCopy(ctx context.Context, f fs.Fs, remote string, src fs.Object, streams int) (newDst fs.Object, err error) {
openWriterAt := f.Features().OpenWriterAt
if openWriterAt == nil {
return nil, errors.New("multi-thread copy: OpenWriterAt not supported")
@@ -136,7 +136,7 @@ func multiThreadCopy(f fs.Fs, remote string, src fs.Object, streams int) (newDst
defer fs.CheckClose(mc.acc, &err)
// create write file handle
mc.wc, err = openWriterAt(remote, mc.size)
mc.wc, err = openWriterAt(ctx, remote, mc.size)
if err != nil {
return nil, errors.Wrap(err, "multpart copy: failed to open destination")
}
@@ -146,7 +146,7 @@ func multiThreadCopy(f fs.Fs, remote string, src fs.Object, streams int) (newDst
for stream := 0; stream < mc.streams; stream++ {
stream := stream
g.Go(func() (err error) {
return mc.copyStream(stream)
return mc.copyStream(ctx, stream)
})
}
err = g.Wait()
@@ -154,12 +154,12 @@ func multiThreadCopy(f fs.Fs, remote string, src fs.Object, streams int) (newDst
return nil, err
}
obj, err := f.NewObject(remote)
obj, err := f.NewObject(ctx, remote)
if err != nil {
return nil, errors.Wrap(err, "multi-thread copy: failed to find object after copy")
}
err = obj.SetModTime(src.ModTime())
err = obj.SetModTime(ctx, src.ModTime(ctx))
switch err {
case nil, fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
default:

9
fs/operations/multithread_test.go
View File

@@ -1,6 +1,7 @@
package operations
import (
"context"
"fmt"
"testing"
@@ -50,20 +51,20 @@ func TestMultithreadCopy(t *testing.T) {
t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) {
contents := fstest.RandomString(test.size)
t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
file1 := r.WriteObject("file1", contents, t1)
file1 := r.WriteObject(context.Background(), "file1", contents, t1)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal)
src, err := r.Fremote.NewObject("file1")
src, err := r.Fremote.NewObject(context.Background(), "file1")
require.NoError(t, err)
dst, err := multiThreadCopy(r.Flocal, "file1", src, 2)
dst, err := multiThreadCopy(context.Background(), r.Flocal, "file1", src, 2)
require.NoError(t, err)
assert.Equal(t, src.Size(), dst.Size())
assert.Equal(t, "file1", dst.Remote())
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
require.NoError(t, dst.Remove())
require.NoError(t, dst.Remove(context.Background()))
})
}

294
fs/operations/operations.go
View File

@@ -45,14 +45,14 @@ import (
// err - may return an error which will already have been logged
//
// If an error is returned it will return equal as false
func CheckHashes(src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error) {
func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error) {
common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
// fs.Debugf(nil, "Shared hashes: %v", common)
if common.Count() == 0 {
return true, hash.None, nil
}
ht = common.GetOne()
srcHash, err := src.Hash(ht)
srcHash, err := src.Hash(ctx, ht)
if err != nil {
fs.CountError(err)
fs.Errorf(src, "Failed to calculate src hash: %v", err)
@@ -61,7 +61,7 @@ func CheckHashes(src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, er
if srcHash == "" {
return true, hash.None, nil
}
dstHash, err := dst.Hash(ht)
dstHash, err := dst.Hash(ctx, ht)
if err != nil {
fs.CountError(err)
fs.Errorf(dst, "Failed to calculate dst hash: %v", err)
@@ -95,8 +95,8 @@ func CheckHashes(src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, er
//
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(src fs.ObjectInfo, dst fs.Object) bool {
return equal(src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
}
// sizeDiffers compare the size of src and dst taking into account the
@@ -110,7 +110,7 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool {
var checksumWarning sync.Once
func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
if sizeDiffers(src, dst) {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
return false
@@ -125,7 +125,7 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
// If checking checksum and not modtime
if checkSum {
// Check the hash
same, ht, _ := CheckHashes(src, dst)
same, ht, _ := CheckHashes(ctx, src, dst)
if !same {
fs.Debugf(src, "%v differ", ht)
return false
@@ -147,8 +147,8 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
fs.Debugf(src, "Sizes identical")
return true
}
srcModTime := src.ModTime()
dstModTime := dst.ModTime()
srcModTime := src.ModTime(ctx)
dstModTime := dst.ModTime(ctx)
dt := dstModTime.Sub(srcModTime)
if dt < modifyWindow && dt > -modifyWindow {
fs.Debugf(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow)
@@ -158,7 +158,7 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
fs.Debugf(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
// Check if the hashes are the same
same, ht, _ := CheckHashes(src, dst)
same, ht, _ := CheckHashes(ctx, src, dst)
if !same {
fs.Debugf(src, "%v differ", ht)
return false
@@ -180,7 +180,7 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
return false
}
// Update the mtime of the dst object here
err := dst.SetModTime(srcModTime)
err := dst.SetModTime(ctx, srcModTime)
if err == fs.ErrorCantSetModTime {
fs.Debugf(dst, "src and dst identical but can't set mod time without re-uploading")
return false
@@ -189,7 +189,7 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
// Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file
// put in the BackupDir than deleted which is what will happen if we don't delete it.
if fs.Config.BackupDir == "" {
err = dst.Remove()
err = dst.Remove(ctx)
if err != nil {
fs.Errorf(dst, "failed to delete before re-upload: %v", err)
}
@@ -209,12 +209,12 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
// Used to remove a failed copy
//
// Returns whether the file was successfully removed or not
func removeFailedCopy(dst fs.Object) bool {
func removeFailedCopy(ctx context.Context, dst fs.Object) bool {
if dst == nil {
return false
}
fs.Infof(dst, "Removing failed copy")
removeErr := dst.Remove()
removeErr := dst.Remove(ctx)
if removeErr != nil {
fs.Infof(dst, "Failed to remove failed copy: %s", removeErr)
return false
@@ -235,9 +235,9 @@ func (o *overrideRemoteObject) Remote() string {
// MimeType returns the mime type of the underlying object or "" if it
// can't be worked out
func (o *overrideRemoteObject) MimeType() string {
func (o *overrideRemoteObject) MimeType(ctx context.Context) string {
if do, ok := o.Object.(fs.MimeTyper); ok {
return do.MimeType()
return do.MimeType(ctx)
}
return ""
}
@@ -250,7 +250,7 @@ var _ fs.MimeTyper = (*overrideRemoteObject)(nil)
//
// It returns the destination object if possible. Note that this may
// be nil.
func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
accounting.Stats.Transferring(src.Remote())
defer func() {
accounting.Stats.DoneTransferring(src.Remote(), err == nil)
@@ -284,7 +284,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
if fs.Config.MaxTransfer >= 0 && accounting.Stats.GetBytes() >= int64(fs.Config.MaxTransfer) {
return nil, accounting.ErrorMaxTransferLimitReached
}
newDst, err = doCopy(src, remote)
newDst, err = doCopy(ctx, src, remote)
if err == nil {
dst = newDst
accounting.Stats.Bytes(dst.Size()) // account the bytes for the server side transfer
@@ -304,7 +304,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
if streams < 2 {
streams = 2
}
dst, err = multiThreadCopy(f, remote, src, int(streams))
dst, err = multiThreadCopy(ctx, f, remote, src, int(streams))
if doUpdate {
actionTaken = "Multi-thread Copied (replaced existing)"
} else {
@@ -312,7 +312,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
}
} else {
var in0 io.ReadCloser
in0, err = newReOpen(src, hashOption, nil, fs.Config.LowLevelRetries)
in0, err = newReOpen(ctx, src, hashOption, nil, fs.Config.LowLevelRetries)
if err != nil {
err = errors.Wrap(err, "failed to open source object")
} else {
@@ -323,7 +323,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
} else {
actionTaken = "Copied (Rcat, new)"
}
dst, err = Rcat(f, remote, in0, src.ModTime())
dst, err = Rcat(ctx, f, remote, in0, src.ModTime(ctx))
newDst = dst
} else {
in := accounting.NewAccount(in0, src).WithBuffer() // account and buffer the transfer
@@ -334,10 +334,10 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
}
if doUpdate {
actionTaken = "Copied (replaced existing)"
err = dst.Update(in, wrappedSrc, hashOption)
err = dst.Update(ctx, in, wrappedSrc, hashOption)
} else {
actionTaken = "Copied (new)"
dst, err = f.Put(in, wrappedSrc, hashOption)
dst, err = f.Put(ctx, in, wrappedSrc, hashOption)
}
closeErr := in.Close()
if err == nil {
@@ -371,20 +371,20 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
fs.Errorf(dst, "%v", err)
fs.CountError(err)
removeFailedCopy(dst)
removeFailedCopy(ctx, dst)
return newDst, err
}
// Verify hashes are the same after transfer - ignoring blank hashes
if !fs.Config.IgnoreChecksum && hashType != hash.None {
var srcSum string
srcSum, err = src.Hash(hashType)
srcSum, err = src.Hash(ctx, hashType)
if err != nil {
fs.CountError(err)
fs.Errorf(src, "Failed to read src hash: %v", err)
} else if srcSum != "" {
var dstSum string
dstSum, err = dst.Hash(hashType)
dstSum, err = dst.Hash(ctx, hashType)
if err != nil {
fs.CountError(err)
fs.Errorf(dst, "Failed to read hash: %v", err)
@@ -392,7 +392,7 @@ func Copy(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Objec
err = errors.Errorf("corrupted on transfer: %v hash differ %q vs %q", hashType, srcSum, dstSum)
fs.Errorf(dst, "%v", err)
fs.CountError(err)
removeFailedCopy(dst)
removeFailedCopy(ctx, dst)
return newDst, err
}
}
@@ -427,7 +427,7 @@ func SameObject(src, dst fs.Object) bool {
//
// It returns the destination object if possible. Note that this may
// be nil.
func Move(fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
accounting.Stats.Checking(src.Remote())
defer func() {
accounting.Stats.DoneChecking(src.Remote())
@@ -441,13 +441,13 @@ func Move(fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Ob
if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && fdst.Features().ServerSideAcrossConfigs)) {
// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
if dst != nil && !SameObject(src, dst) {
err = DeleteFile(dst)
err = DeleteFile(ctx, dst)
if err != nil {
return newDst, err
}
}
// Move dst <- src
newDst, err = doMove(src, remote)
newDst, err = doMove(ctx, src, remote)
switch err {
case nil:
fs.Infof(src, "Moved (server side)")
@@ -461,13 +461,13 @@ func Move(fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Ob
}
}
// Move not found or didn't work so copy dst <- src
newDst, err = Copy(fdst, dst, remote, src)
newDst, err = Copy(ctx, fdst, dst, remote, src)
if err != nil {
fs.Errorf(src, "Not deleting source as copy failed: %v", err)
return newDst, err
}
// Delete src if no error on copy
return newDst, DeleteFile(src)
return newDst, DeleteFile(ctx, src)
}
// CanServerSideMove returns true if fdst support server side moves or
@@ -500,7 +500,7 @@ func SuffixName(remote string) string {
//
// If backupDir is set then it moves the file to there instead of
// deleting
func DeleteFileWithBackupDir(dst fs.Object, backupDir fs.Fs) (err error) {
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
accounting.Stats.Checking(dst.Remote())
numDeletes := accounting.Stats.Deletes(1)
if fs.Config.MaxDelete != -1 && numDeletes > fs.Config.MaxDelete {
@@ -517,11 +517,11 @@ func DeleteFileWithBackupDir(dst fs.Object, backupDir fs.Fs) (err error) {
err = errors.New("parameter to --backup-dir has to be on the same remote as destination")
} else {
remoteWithSuffix := SuffixName(dst.Remote())
overwritten, _ := backupDir.NewObject(remoteWithSuffix)
_, err = Move(backupDir, overwritten, remoteWithSuffix, dst)
overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix)
_, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst)
}
} else {
err = dst.Remove()
err = dst.Remove(ctx)
}
if err != nil {
fs.CountError(err)
@@ -537,8 +537,8 @@ func DeleteFileWithBackupDir(dst fs.Object, backupDir fs.Fs) (err error) {
//
// If useBackupDir is set and --backup-dir is in effect then it moves
// the file to there instead of deleting
func DeleteFile(dst fs.Object) (err error) {
return DeleteFileWithBackupDir(dst, nil)
func DeleteFile(ctx context.Context, dst fs.Object) (err error) {
return DeleteFileWithBackupDir(ctx, dst, nil)
}
// DeleteFilesWithBackupDir removes all the files passed in the
@@ -546,7 +546,7 @@ func DeleteFile(dst fs.Object) (err error) {
//
// If backupDir is set the files will be placed into that directory
// instead of being deleted.
func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error {
func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error {
var wg sync.WaitGroup
wg.Add(fs.Config.Transfers)
var errorCount int32
@@ -556,7 +556,7 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error
go func() {
defer wg.Done()
for dst := range toBeDeleted {
err := DeleteFileWithBackupDir(dst, backupDir)
err := DeleteFileWithBackupDir(ctx, dst, backupDir)
if err != nil {
atomic.AddInt32(&errorCount, 1)
if fserrors.IsFatalError(err) {
@@ -581,8 +581,8 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error
}
// DeleteFiles removes all the files passed in the channel
func DeleteFiles(toBeDeleted fs.ObjectsChan) error {
return DeleteFilesWithBackupDir(toBeDeleted, nil)
func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error {
return DeleteFilesWithBackupDir(ctx, toBeDeleted, nil)
}
// SameRemoteType returns true if fdst and fsrc are the same type
@@ -624,8 +624,8 @@ func Overlapping(fdst, fsrc fs.Info) bool {
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
same, ht, err := CheckHashes(src, dst)
func checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
same, ht, err := CheckHashes(ctx, src, dst)
if err != nil {
// CheckHashes will log and count errors
return true, false
@@ -643,7 +643,7 @@ func checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
}
// checkFn is the the type of the checking function used in CheckFn()
type checkFn func(a, b fs.Object) (differ bool, noHash bool)
type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool)
// checkMarch is used to march over two Fses in the same way as
// sync/copy
@@ -698,7 +698,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
}
// check to see if two objects are identical using the check function
func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash bool) {
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
accounting.Stats.Checking(src.Remote())
defer accounting.Stats.DoneChecking(src.Remote())
if sizeDiffers(src, dst) {
@@ -710,16 +710,16 @@ func (c *checkMarch) checkIdentical(dst, src fs.Object) (differ bool, noHash boo
if fs.Config.SizeOnly {
return false, false
}
return c.check(dst, src)
return c.check(ctx, dst, src)
}
// Match is called when src and dst are present, so sync src to dst
func (c *checkMarch) Match(dst, src fs.DirEntry) (recurse bool) {
func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
switch srcX := src.(type) {
case fs.Object:
dstX, ok := dst.(fs.Object)
if ok {
differ, noHash := c.checkIdentical(dstX, srcX)
differ, noHash := c.checkIdentical(ctx, dstX, srcX)
if differ {
atomic.AddInt32(&c.differences, 1)
} else {
@@ -761,7 +761,7 @@ func (c *checkMarch) Match(dst, src fs.DirEntry) (recurse bool) {
//
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func CheckFn(fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
func CheckFn(ctx context.Context, fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
c := &checkMarch{
fdst: fdst,
fsrc: fsrc,
@@ -771,7 +771,7 @@ func CheckFn(fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
// set up a march over fdst and fsrc
m := &march.March{
Ctx: context.Background(),
Ctx: ctx,
Fdst: fdst,
Fsrc: fsrc,
Dir: "",
@@ -801,8 +801,8 @@ func CheckFn(fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
}
// Check the files in fsrc and fdst according to Size and hash
func Check(fdst, fsrc fs.Fs, oneway bool) error {
return CheckFn(fdst, fsrc, checkIdentical, oneway)
func Check(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
return CheckFn(ctx, fdst, fsrc, checkIdentical, oneway)
}
// CheckEqualReaders checks to see if in1 and in2 have the same
@@ -839,15 +839,15 @@ func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) {
// reading all their bytes if necessary.
//
// it returns true if differences were found
func CheckIdentical(dst, src fs.Object) (differ bool, err error) {
in1, err := dst.Open()
func CheckIdentical(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
in1, err := dst.Open(ctx)
if err != nil {
return true, errors.Wrapf(err, "failed to open %q", dst)
}
in1 = accounting.NewAccount(in1, dst).WithBuffer() // account and buffer the transfer
defer fs.CheckClose(in1, &err)
in2, err := src.Open()
in2, err := src.Open(ctx)
if err != nil {
return true, errors.Wrapf(err, "failed to open %q", src)
}
@@ -859,9 +859,9 @@ func CheckIdentical(dst, src fs.Object) (differ bool, err error) {
// CheckDownload checks the files in fsrc and fdst according to Size
// and the actual contents of the files.
func CheckDownload(fdst, fsrc fs.Fs, oneway bool) error {
check := func(a, b fs.Object) (differ bool, noHash bool) {
differ, err := CheckIdentical(a, b)
func CheckDownload(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error {
check := func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool) {
differ, err := CheckIdentical(ctx, a, b)
if err != nil {
fs.CountError(err)
fs.Errorf(a, "Failed to download: %v", err)
@@ -869,14 +869,14 @@ func CheckDownload(fdst, fsrc fs.Fs, oneway bool) error {
}
return differ, false
}
return CheckFn(fdst, fsrc, check, oneway)
return CheckFn(ctx, fdst, fsrc, check, oneway)
}
// ListFn lists the Fs to the supplied function
//
// Lists in parallel which may get them out of order
func ListFn(f fs.Fs, fn func(fs.Object)) error {
return walk.ListR(f, "", false, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error {
return walk.ListR(ctx, f, "", false, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(fn)
return nil
})
@@ -899,8 +899,8 @@ func syncFprintf(w io.Writer, format string, a ...interface{}) {
// Shows size and path - obeys includes and excludes
//
// Lists in parallel which may get them out of order
func List(f fs.Fs, w io.Writer) error {
return ListFn(f, func(o fs.Object) {
func List(ctx context.Context, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
syncFprintf(w, "%9d %s\n", o.Size(), o.Remote())
})
}
@@ -910,10 +910,10 @@ func List(f fs.Fs, w io.Writer) error {
// Shows size, mod time and path - obeys includes and excludes
//
// Lists in parallel which may get them out of order
func ListLong(f fs.Fs, w io.Writer) error {
return ListFn(f, func(o fs.Object) {
func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
accounting.Stats.Checking(o.Remote())
modTime := o.ModTime()
modTime := o.ModTime(ctx)
accounting.Stats.DoneChecking(o.Remote())
syncFprintf(w, "%9d %s %s\n", o.Size(), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote())
})
@@ -925,8 +925,8 @@ func ListLong(f fs.Fs, w io.Writer) error {
// excludes
//
// Lists in parallel which may get them out of order
func Md5sum(f fs.Fs, w io.Writer) error {
return HashLister(hash.MD5, f, w)
func Md5sum(ctx context.Context, f fs.Fs, w io.Writer) error {
return HashLister(ctx, hash.MD5, f, w)
}
// Sha1sum list the Fs to the supplied writer
@@ -934,8 +934,8 @@ func Md5sum(f fs.Fs, w io.Writer) error {
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func Sha1sum(f fs.Fs, w io.Writer) error {
return HashLister(hash.SHA1, f, w)
func Sha1sum(ctx context.Context, f fs.Fs, w io.Writer) error {
return HashLister(ctx, hash.SHA1, f, w)
}
// DropboxHashSum list the Fs to the supplied writer
@@ -943,15 +943,15 @@ func Sha1sum(f fs.Fs, w io.Writer) error {
// Obeys includes and excludes
//
// Lists in parallel which may get them out of order
func DropboxHashSum(f fs.Fs, w io.Writer) error {
return HashLister(hash.Dropbox, f, w)
func DropboxHashSum(ctx context.Context, f fs.Fs, w io.Writer) error {
return HashLister(ctx, hash.Dropbox, f, w)
}
// hashSum returns the human readable hash for ht passed in. This may
// be UNSUPPORTED or ERROR.
func hashSum(ht hash.Type, o fs.Object) string {
func hashSum(ctx context.Context, ht hash.Type, o fs.Object) string {
accounting.Stats.Checking(o.Remote())
sum, err := o.Hash(ht)
sum, err := o.Hash(ctx, ht)
accounting.Stats.DoneChecking(o.Remote())
if err == hash.ErrUnsupported {
sum = "UNSUPPORTED"
@@ -963,9 +963,9 @@ func hashSum(ht hash.Type, o fs.Object) string {
}
// HashLister does a md5sum equivalent for the hash type passed in
func HashLister(ht hash.Type, f fs.Fs, w io.Writer) error {
return ListFn(f, func(o fs.Object) {
sum := hashSum(ht, o)
func HashLister(ctx context.Context, ht hash.Type, f fs.Fs, w io.Writer) error {
return ListFn(ctx, f, func(o fs.Object) {
sum := hashSum(ctx, ht, o)
syncFprintf(w, "%*s %s\n", hash.Width[ht], sum, o.Remote())
})
}
@@ -973,8 +973,8 @@ func HashLister(ht hash.Type, f fs.Fs, w io.Writer) error {
// Count counts the objects and their sizes in the Fs
//
// Obeys includes and excludes
func Count(f fs.Fs) (objects int64, size int64, err error) {
err = ListFn(f, func(o fs.Object) {
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
err = ListFn(ctx, f, func(o fs.Object) {
atomic.AddInt64(&objects, 1)
objectSize := o.Size()
if objectSize > 0 {
@@ -994,11 +994,11 @@ func ConfigMaxDepth(recursive bool) int {
}
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
func ListDir(f fs.Fs, w io.Writer) error {
return walk.ListR(f, "", false, ConfigMaxDepth(false), walk.ListDirs, func(entries fs.DirEntries) error {
func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
return walk.ListR(ctx, f, "", false, ConfigMaxDepth(false), walk.ListDirs, func(entries fs.DirEntries) error {
entries.ForDir(func(dir fs.Directory) {
if dir != nil {
syncFprintf(w, "%12d %13s %9d %s\n", dir.Size(), dir.ModTime().Local().Format("2006-01-02 15:04:05"), dir.Items(), dir.Remote())
syncFprintf(w, "%12d %13s %9d %s\n", dir.Size(), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), dir.Items(), dir.Remote())
}
})
return nil
@@ -1006,13 +1006,13 @@ func ListDir(f fs.Fs, w io.Writer) error {
}
// Mkdir makes a destination directory or container
func Mkdir(f fs.Fs, dir string) error {
func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
if fs.Config.DryRun {
fs.Logf(fs.LogDirName(f, dir), "Not making directory as dry run is set")
return nil
}
fs.Debugf(fs.LogDirName(f, dir), "Making directory")
err := f.Mkdir(dir)
err := f.Mkdir(ctx, dir)
if err != nil {
fs.CountError(err)
return err
@@ -1022,18 +1022,18 @@ func Mkdir(f fs.Fs, dir string) error {
// TryRmdir removes a container but not if not empty. It doesn't
// count errors but may return one.
func TryRmdir(f fs.Fs, dir string) error {
func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
if fs.Config.DryRun {
fs.Logf(fs.LogDirName(f, dir), "Not deleting as dry run is set")
return nil
}
fs.Debugf(fs.LogDirName(f, dir), "Removing directory")
return f.Rmdir(dir)
return f.Rmdir(ctx, dir)
}
// Rmdir removes a container but not if not empty
func Rmdir(f fs.Fs, dir string) error {
err := TryRmdir(f, dir)
func Rmdir(ctx context.Context, f fs.Fs, dir string) error {
err := TryRmdir(ctx, f, dir)
if err != nil {
fs.CountError(err)
return err
@@ -1042,7 +1042,7 @@ func Rmdir(f fs.Fs, dir string) error {
}
// Purge removes a directory and all of its contents
func Purge(f fs.Fs, dir string) error {
func Purge(ctx context.Context, f fs.Fs, dir string) error {
doFallbackPurge := true
var err error
if dir == "" {
@@ -1052,7 +1052,7 @@ func Purge(f fs.Fs, dir string) error {
if fs.Config.DryRun {
fs.Logf(f, "Not purging as --dry-run set")
} else {
err = doPurge()
err = doPurge(ctx)
if err == fs.ErrorCantPurge {
doFallbackPurge = true
}
@@ -1061,11 +1061,11 @@ func Purge(f fs.Fs, dir string) error {
}
if doFallbackPurge {
// DeleteFiles and Rmdir observe --dry-run
err = DeleteFiles(listToChan(f, dir))
err = DeleteFiles(ctx, listToChan(ctx, f, dir))
if err != nil {
return err
}
err = Rmdirs(f, dir, false)
err = Rmdirs(ctx, f, dir, false)
}
if err != nil {
fs.CountError(err)
@@ -1076,13 +1076,13 @@ func Purge(f fs.Fs, dir string) error {
// Delete removes all the contents of a container. Unlike Purge, it
// obeys includes and excludes.
func Delete(f fs.Fs) error {
func Delete(ctx context.Context, f fs.Fs) error {
delChan := make(fs.ObjectsChan, fs.Config.Transfers)
delErr := make(chan error, 1)
go func() {
delErr <- DeleteFiles(delChan)
delErr <- DeleteFiles(ctx, delChan)
}()
err := ListFn(f, func(o fs.Object) {
err := ListFn(ctx, f, func(o fs.Object) {
delChan <- o
})
close(delChan)
@@ -1099,11 +1099,11 @@ func Delete(f fs.Fs) error {
// channel.
//
// If the error was ErrorDirNotFound then it will be ignored
func listToChan(f fs.Fs, dir string) fs.ObjectsChan {
func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {
o := make(fs.ObjectsChan, fs.Config.Checkers)
go func() {
defer close(o)
err := walk.ListR(f, dir, true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
err := walk.ListR(ctx, f, dir, true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(obj fs.Object) {
o <- obj
})
@@ -1119,7 +1119,7 @@ func listToChan(f fs.Fs, dir string) fs.ObjectsChan {
}
// CleanUp removes the trash for the Fs
func CleanUp(f fs.Fs) error {
func CleanUp(ctx context.Context, f fs.Fs) error {
doCleanUp := f.Features().CleanUp
if doCleanUp == nil {
return errors.Errorf("%v doesn't support cleanup", f)
@@ -1128,7 +1128,7 @@ func CleanUp(f fs.Fs) error {
fs.Logf(f, "Not running cleanup as --dry-run set")
return nil
}
return doCleanUp()
return doCleanUp(ctx)
}
// wrap a Reader and a Closer together into a ReadCloser
@@ -1145,9 +1145,9 @@ type readCloser struct {
//
// if count < 0 then it will be ignored
// if count >= 0 then only that many characters will be output
func Cat(f fs.Fs, w io.Writer, offset, count int64) error {
func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
var mu sync.Mutex
return ListFn(f, func(o fs.Object) {
return ListFn(ctx, f, func(o fs.Object) {
var err error
accounting.Stats.Transferring(o.Remote())
defer func() {
@@ -1165,7 +1165,7 @@ func Cat(f fs.Fs, w io.Writer, offset, count int64) error {
if opt.Start > 0 || opt.End >= 0 {
options = append(options, &opt)
}
in, err := o.Open(options...)
in, err := o.Open(ctx, options...)
if err != nil {
fs.CountError(err)
fs.Errorf(o, "Failed to open: %v", err)
@@ -1198,7 +1198,7 @@ func Cat(f fs.Fs, w io.Writer, offset, count int64) error {
}
// Rcat reads data from the Reader until EOF and uploads it to a file on remote
func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error) {
func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error) {
accounting.Stats.Transferring(dstFileName)
in = accounting.NewAccountSizeName(in, -1, dstFileName).WithBuffer()
defer func() {
@@ -1218,7 +1218,7 @@ func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (
compare := func(dst fs.Object) error {
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, hash.Sums(), fdst)
if !Equal(src, dst) {
if !Equal(ctx, src, dst) {
err = errors.Errorf("corrupted on transfer")
fs.CountError(err)
fs.Errorf(dst, "%v", err)
@@ -1232,7 +1232,7 @@ func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (
if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", n)
src := object.NewMemoryObject(dstFileName, modTime, buf[:n])
return Copy(fdst, nil, dstFileName, src)
return Copy(ctx, fdst, nil, dstFileName, src)
}
// Make a new ReadCloser with the bits we've already read
@@ -1250,7 +1250,7 @@ func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (
return nil, errors.Wrap(err, "Failed to create temporary local FS to spool file")
}
defer func() {
err := Purge(tmpLocalFs, "")
err := Purge(ctx, tmpLocalFs, "")
if err != nil {
fs.Infof(tmpLocalFs, "Failed to cleanup temporary FS: %v", err)
}
@@ -1266,7 +1266,7 @@ func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (
}
objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
if dst, err = fStreamTo.Features().PutStream(in, objInfo, hashOption); err != nil {
if dst, err = fStreamTo.Features().PutStream(ctx, in, objInfo, hashOption); err != nil {
return dst, err
}
if err = compare(dst); err != nil {
@@ -1274,26 +1274,26 @@ func Rcat(fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (
}
if !canStream {
// copy dst (which is the local object we have just streamed to) to the remote
return Copy(fdst, nil, dstFileName, dst)
return Copy(ctx, fdst, nil, dstFileName, dst)
}
return dst, nil
}
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
func PublicLink(f fs.Fs, remote string) (string, error) {
func PublicLink(ctx context.Context, f fs.Fs, remote string) (string, error) {
doPublicLink := f.Features().PublicLink
if doPublicLink == nil {
return "", errors.Errorf("%v doesn't support public links", f)
}
return doPublicLink(remote)
return doPublicLink(ctx, remote)
}
// Rmdirs removes any empty directories (or directories only
// containing empty directories) under f, including f.
func Rmdirs(f fs.Fs, dir string, leaveRoot bool) error {
func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
dirEmpty := make(map[string]bool)
dirEmpty[dir] = !leaveRoot
err := walk.Walk(f, dir, true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
err := walk.Walk(ctx, f, dir, true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
if err != nil {
fs.CountError(err)
fs.Errorf(f, "Failed to list %q: %v", dirPath, err)
@@ -1340,7 +1340,7 @@ func Rmdirs(f fs.Fs, dir string, leaveRoot bool) error {
sort.Strings(toDelete)
for i := len(toDelete) - 1; i >= 0; i-- {
dir := toDelete[i]
err := TryRmdir(f, dir)
err := TryRmdir(ctx, f, dir)
if err != nil {
fs.CountError(err)
fs.Errorf(dir, "Failed to rmdir: %v", err)
@@ -1355,7 +1355,7 @@ func Rmdirs(f fs.Fs, dir string, leaveRoot bool) error {
//
// Returns a flag which indicates whether the file needs to be
// transferred or not.
func NeedTransfer(dst, src fs.Object) bool {
func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
if dst == nil {
fs.Debugf(src, "Couldn't find file - need to transfer")
return true
@@ -1372,8 +1372,8 @@ func NeedTransfer(dst, src fs.Object) bool {
}
// If UpdateOlder is in effect, skip if dst is newer than src
if fs.Config.UpdateOlder {
srcModTime := src.ModTime()
dstModTime := dst.ModTime()
srcModTime := src.ModTime(ctx)
dstModTime := dst.ModTime(ctx)
dt := dstModTime.Sub(srcModTime)
// If have a mutually agreed precision then use that
modifyWindow := fs.GetModifyWindow(dst.Fs(), src.Fs())
@@ -1398,7 +1398,7 @@ func NeedTransfer(dst, src fs.Object) bool {
}
} else {
// Check to see if changed or not
if Equal(src, dst) {
if Equal(ctx, src, dst) {
fs.Debugf(src, "Unchanged skipping")
return false
}
@@ -1408,7 +1408,7 @@ func NeedTransfer(dst, src fs.Object) bool {
// RcatSize reads data from the Reader until EOF and uploads it to a file on remote.
// Pass in size >=0 if known, <0 if not known
func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error) {
func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error) {
var obj fs.Object
if size >= 0 {
@@ -1434,7 +1434,7 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
accounting.Stats.DoneTransferring(dstFileName, err == nil)
}()
info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst)
obj, err = fdst.Put(in, info)
obj, err = fdst.Put(ctx, in, info)
if err != nil {
fs.Errorf(dstFileName, "Post request put error: %v", err)
@@ -1442,7 +1442,7 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
}
} else {
// Size unknown use Rcat
obj, err = Rcat(fdst, dstFileName, in, modTime)
obj, err = Rcat(ctx, fdst, dstFileName, in, modTime)
if err != nil {
fs.Errorf(dstFileName, "Post request rcat error: %v", err)
@@ -1454,7 +1454,7 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
}
// CopyURL copies the data from the url to (fdst, dstFileName)
func CopyURL(fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
client := fshttp.NewClient(fs.Config)
resp, err := client.Get(url)
@@ -1462,11 +1462,11 @@ func CopyURL(fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err err
return nil, err
}
defer fs.CheckClose(resp.Body, &err)
return RcatSize(fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
}
// moveOrCopyFile moves or copies a single file possibly to a new name
func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
dstFilePath := path.Join(fdst.Root(), dstFileName)
srcFilePath := path.Join(fsrc.Root(), srcFileName)
if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {
@@ -1481,13 +1481,13 @@ func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName stri
}
// Find src object
srcObj, err := fsrc.NewObject(srcFileName)
srcObj, err := fsrc.NewObject(ctx, srcFileName)
if err != nil {
return err
}
// Find dst object if it exists
dstObj, err := fdst.NewObject(dstFileName)
dstObj, err := fdst.NewObject(ctx, dstFileName)
if err == fs.ErrorObjectNotFound {
dstObj = nil
} else if err != nil {
@@ -1501,7 +1501,7 @@ func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName stri
if !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.ToLower(dstFilePath) == strings.ToLower(srcFilePath) {
// Create random name to temporarily move file to
tmpObjName := dstFileName + "-rclone-move-" + random(8)
_, err := fdst.NewObject(tmpObjName)
_, err := fdst.NewObject(ctx, tmpObjName)
if err != fs.ErrorObjectNotFound {
if err == nil {
return errors.New("found an already existing file with a randomly generated name. Try the operation again")
@@ -1509,17 +1509,17 @@ func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName stri
return errors.Wrap(err, "error while attempting to move file to a temporary location")
}
accounting.Stats.Transferring(srcFileName)
tmpObj, err := Op(fdst, nil, tmpObjName, srcObj)
tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj)
if err != nil {
accounting.Stats.DoneTransferring(srcFileName, false)
return errors.Wrap(err, "error while moving file to temporary location")
}
_, err = Op(fdst, nil, dstFileName, tmpObj)
_, err = Op(ctx, fdst, nil, dstFileName, tmpObj)
accounting.Stats.DoneTransferring(srcFileName, err == nil)
return err
}
if NeedTransfer(dstObj, srcObj) {
if NeedTransfer(ctx, dstObj, srcObj) {
// If destination already exists, then we must move it into --backup-dir if required
if dstObj != nil && fs.Config.BackupDir != "" {
backupDir, err := cache.Get(fs.Config.BackupDir)
@@ -1527,8 +1527,8 @@ func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName stri
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
remoteWithSuffix := SuffixName(dstObj.Remote())
overwritten, _ := backupDir.NewObject(remoteWithSuffix)
_, err = Move(backupDir, overwritten, remoteWithSuffix, dstObj)
overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix)
_, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dstObj)
if err != nil {
return errors.Wrap(err, "moving to --backup-dir failed")
}
@@ -1536,11 +1536,11 @@ func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName stri
dstObj = nil
}
_, err = Op(fdst, dstObj, dstFileName, srcObj)
_, err = Op(ctx, fdst, dstObj, dstFileName, srcObj)
} else {
accounting.Stats.Checking(srcFileName)
if !cp {
err = DeleteFile(srcObj)
err = DeleteFile(ctx, srcObj)
}
defer accounting.Stats.DoneChecking(srcFileName)
}
@@ -1559,18 +1559,18 @@ func random(length int) string {
}
// MoveFile moves a single file possibly to a new name
func MoveFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, false)
func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false)
}
// CopyFile moves a single file possibly to a new name
func CopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, true)
func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true)
}
// SetTier changes tier of object in remote
func SetTier(fsrc fs.Fs, tier string) error {
return ListFn(fsrc, func(o fs.Object) {
func SetTier(ctx context.Context, fsrc fs.Fs, tier string) error {
return ListFn(ctx, fsrc, func(o fs.Object) {
objImpl, ok := o.(fs.SetTierer)
if !ok {
fs.Errorf(fsrc, "Remote object does not implement SetTier")
@@ -1732,14 +1732,14 @@ func (l *ListFormat) Format(entry *ListJSONItem) (result string) {
//
// It does this by loading the directory tree into memory (using ListR
// if available) and doing renames in parallel.
func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
// Use DirMove if possible
if doDirMove := f.Features().DirMove; doDirMove != nil {
return doDirMove(f, srcRemote, dstRemote)
return doDirMove(ctx, f, srcRemote, dstRemote)
}
// Load the directory tree into memory
tree, err := walk.NewDirTree(f, srcRemote, true, -1)
tree, err := walk.NewDirTree(ctx, f, srcRemote, true, -1)
if err != nil {
return errors.Wrap(err, "RenameDir tree walk")
}
@@ -1750,7 +1750,7 @@ func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
// Make the destination directories - must be done in order not in parallel
for _, dir := range dirs {
dstPath := dstRemote + dir[len(srcRemote):]
err := f.Mkdir(dstPath)
err := f.Mkdir(ctx, dstPath)
if err != nil {
return errors.Wrap(err, "RenameDir mkdir")
}
@@ -1766,8 +1766,8 @@ func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
for i := 0; i < fs.Config.Transfers; i++ {
g.Go(func() error {
for job := range renames {
dstOverwritten, _ := f.NewObject(job.newPath)
_, err := Move(f, dstOverwritten, job.newPath, job.o)
dstOverwritten, _ := f.NewObject(ctx, job.newPath)
_, err := Move(ctx, f, dstOverwritten, job.newPath, job.o)
if err != nil {
return err
}
@@ -1797,7 +1797,7 @@ func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
// Remove the source directories in reverse order
for i := len(dirs) - 1; i >= 0; i-- {
err := f.Rmdir(dirs[i])
err := f.Rmdir(ctx, dirs[i])
if err != nil {
return errors.Wrap(err, "RenameDir rmdir")
}

201
fs/operations/operations_test.go
View File

@@ -21,6 +21,7 @@ package operations_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@@ -62,23 +63,23 @@ func TestMkdir(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
err := operations.Mkdir(r.Fremote, "")
err := operations.Mkdir(context.Background(), r.Fremote, "")
require.NoError(t, err)
fstest.CheckListing(t, r.Fremote, []fstest.Item{})
err = operations.Mkdir(r.Fremote, "")
err = operations.Mkdir(context.Background(), r.Fremote, "")
require.NoError(t, err)
}
func TestLsd(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("sub dir/hello world", "hello world", t1)
file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)
fstest.CheckItems(t, r.Fremote, file1)
var buf bytes.Buffer
err := operations.ListDir(r.Fremote, &buf)
err := operations.ListDir(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res := buf.String()
assert.Contains(t, res, "sub dir\n")
@@ -87,13 +88,13 @@ func TestLsd(t *testing.T) {
func TestLs(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
var buf bytes.Buffer
err := operations.List(r.Fremote, &buf)
err := operations.List(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res := buf.String()
assert.Contains(t, res, " 0 empty space\n")
@@ -103,8 +104,8 @@ func TestLs(t *testing.T) {
func TestLsWithFilesFrom(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
@@ -122,7 +123,7 @@ func TestLsWithFilesFrom(t *testing.T) {
}()
var buf bytes.Buffer
err = operations.List(r.Fremote, &buf)
err = operations.List(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
assert.Equal(t, " 60 potato2\n", buf.String())
@@ -134,7 +135,7 @@ func TestLsWithFilesFrom(t *testing.T) {
}()
buf.Reset()
err = operations.List(r.Fremote, &buf)
err = operations.List(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
assert.Equal(t, " 60 potato2\n", buf.String())
}
@@ -142,13 +143,13 @@ func TestLsWithFilesFrom(t *testing.T) {
func TestLsLong(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
var buf bytes.Buffer
err := operations.ListLong(r.Fremote, &buf)
err := operations.ListLong(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res := buf.String()
lines := strings.Split(strings.Trim(res, "\n"), "\n")
@@ -187,15 +188,15 @@ func TestLsLong(t *testing.T) {
func TestHashSums(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
// MD5 Sum
var buf bytes.Buffer
err := operations.Md5sum(r.Fremote, &buf)
err := operations.Md5sum(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res := buf.String()
if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e empty space\n") &&
@@ -212,7 +213,7 @@ func TestHashSums(t *testing.T) {
// SHA1 Sum
buf.Reset()
err = operations.Sha1sum(r.Fremote, &buf)
err = operations.Sha1sum(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res = buf.String()
if !strings.Contains(res, "da39a3ee5e6b4b0d3255bfef95601890afd80709 empty space\n") &&
@@ -229,7 +230,7 @@ func TestHashSums(t *testing.T) {
// Dropbox Hash Sum
buf.Reset()
err = operations.DropboxHashSum(r.Fremote, &buf)
err = operations.DropboxHashSum(context.Background(), r.Fremote, &buf)
require.NoError(t, err)
res = buf.String()
if !strings.Contains(res, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 empty space\n") &&
@@ -274,9 +275,9 @@ func TestSuffixName(t *testing.T) {
func TestCount(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file3 := r.WriteBoth("sub dir/potato3", "hello", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
file3 := r.WriteBoth(context.Background(), "sub dir/potato3", "hello", t2)
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
@@ -284,7 +285,7 @@ func TestCount(t *testing.T) {
fs.Config.MaxDepth = 1
defer func() { fs.Config.MaxDepth = -1 }()
objects, size, err := operations.Count(r.Fremote)
objects, size, err := operations.Count(context.Background(), r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(2), objects)
assert.Equal(t, int64(60), size)
@@ -293,9 +294,9 @@ func TestCount(t *testing.T) {
func TestDelete(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject("large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(context.Background(), "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
filter.Active.Opt.MaxSize = 60
@@ -303,12 +304,12 @@ func TestDelete(t *testing.T) {
filter.Active.Opt.MaxSize = -1
}()
err := operations.Delete(r.Fremote)
err := operations.Delete(context.Background(), r.Fremote)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file3)
}
func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) error) {
func testCheck(t *testing.T, checkFunction func(ctx context.Context, fdst, fsrc fs.Fs, oneway bool) error) {
r := fstest.NewRun(t)
defer r.Finalise()
@@ -320,7 +321,7 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
defer func() {
log.SetOutput(os.Stderr)
}()
err := checkFunction(r.Fremote, r.Flocal, oneway)
err := checkFunction(context.Background(), r.Fremote, r.Flocal, oneway)
gotErrors := accounting.Stats.GetErrors()
gotChecks := accounting.Stats.GetChecks()
if wantErrors == 0 && err != nil {
@@ -341,7 +342,7 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
fs.Debugf(r.Fremote, "%d: Ending check test", i)
}
file1 := r.WriteBoth("rutabaga", "is tasty", t3)
file1 := r.WriteBoth(context.Background(), "rutabaga", "is tasty", t3)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file1)
check(1, 0, 1, false)
@@ -350,15 +351,15 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
fstest.CheckItems(t, r.Flocal, file1, file2)
check(2, 1, 1, false)
file3 := r.WriteObject("empty space", "", t2)
file3 := r.WriteObject(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file1, file3)
check(3, 2, 1, false)
file2r := file2
if fs.Config.SizeOnly {
file2r = r.WriteObject("potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
file2r = r.WriteObject(context.Background(), "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
} else {
r.WriteObject("potato2", "------------------------------------------------------------", t1)
r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
}
fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
check(4, 1, 2, false)
@@ -367,7 +368,7 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
check(5, 0, 3, false)
file4 := r.WriteObject("remotepotato", "------------------------------------------------------------", t1)
file4 := r.WriteObject(context.Background(), "remotepotato", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.Fremote, file1, file2r, file3, file4)
check(6, 1, 3, false)
check(7, 0, 3, true)
@@ -390,8 +391,8 @@ func TestCheckSizeOnly(t *testing.T) {
func TestCat(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("file1", "ABCDEFGHIJ", t1)
file2 := r.WriteBoth("file2", "012345678", t2)
file1 := r.WriteBoth(context.Background(), "file1", "ABCDEFGHIJ", t1)
file2 := r.WriteBoth(context.Background(), "file2", "012345678", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
@@ -407,7 +408,7 @@ func TestCat(t *testing.T) {
{1, 3, "BCD", "123"},
} {
var buf bytes.Buffer
err := operations.Cat(r.Fremote, &buf, test.offset, test.count)
err := operations.Cat(context.Background(), r.Fremote, &buf, test.offset, test.count)
require.NoError(t, err)
res := buf.String()
@@ -440,11 +441,11 @@ func TestRcat(t *testing.T) {
path2 := prefix + "big_file_from_pipe"
in := ioutil.NopCloser(strings.NewReader(data1))
_, err := operations.Rcat(r.Fremote, path1, in, t1)
_, err := operations.Rcat(context.Background(), r.Fremote, path1, in, t1)
require.NoError(t, err)
in = ioutil.NopCloser(strings.NewReader(data2))
_, err = operations.Rcat(r.Fremote, path2, in, t2)
_, err = operations.Rcat(context.Background(), r.Fremote, path2, in, t2)
require.NoError(t, err)
file1 := fstest.NewItem(path1, data1, t1)
@@ -459,21 +460,21 @@ func TestRcat(t *testing.T) {
func TestPurge(t *testing.T) {
r := fstest.NewRunIndividual(t) // make new container (azureblob has delayed mkdir after rmdir)
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
// Make some files and dirs
r.ForceMkdir(r.Fremote)
file1 := r.WriteObject("A1/B1/C1/one", "aaa", t1)
r.ForceMkdir(context.Background(), r.Fremote)
file1 := r.WriteObject(context.Background(), "A1/B1/C1/one", "aaa", t1)
//..and dirs we expect to delete
require.NoError(t, operations.Mkdir(r.Fremote, "A2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2/C2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3/B3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3/B3/C4"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B2/C2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B1/C3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3/B3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3/B3/C4"))
//..and one more file at the end
file2 := r.WriteObject("A1/two", "bbb", t2)
file2 := r.WriteObject(context.Background(), "A1/two", "bbb", t2)
fstest.CheckListingWithPrecision(
t,
@@ -496,7 +497,7 @@ func TestPurge(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.Purge(r.Fremote, "A1/B1"))
require.NoError(t, operations.Purge(context.Background(), r.Fremote, "A1/B1"))
fstest.CheckListingWithPrecision(
t,
@@ -516,7 +517,7 @@ func TestPurge(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.Purge(r.Fremote, ""))
require.NoError(t, operations.Purge(context.Background(), r.Fremote, ""))
fstest.CheckListingWithPrecision(
t,
@@ -531,21 +532,21 @@ func TestPurge(t *testing.T) {
func TestRmdirsNoLeaveRoot(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
// Make some files and dirs we expect to keep
r.ForceMkdir(r.Fremote)
file1 := r.WriteObject("A1/B1/C1/one", "aaa", t1)
r.ForceMkdir(context.Background(), r.Fremote)
file1 := r.WriteObject(context.Background(), "A1/B1/C1/one", "aaa", t1)
//..and dirs we expect to delete
require.NoError(t, operations.Mkdir(r.Fremote, "A2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2/C2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3/B3"))
require.NoError(t, operations.Mkdir(r.Fremote, "A3/B3/C4"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B2/C2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B1/C3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3/B3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A3/B3/C4"))
//..and one more file at the end
file2 := r.WriteObject("A1/two", "bbb", t2)
file2 := r.WriteObject(context.Background(), "A1/two", "bbb", t2)
fstest.CheckListingWithPrecision(
t,
@@ -568,7 +569,7 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.Rmdirs(r.Fremote, "A3/B3/C4", false))
require.NoError(t, operations.Rmdirs(context.Background(), r.Fremote, "A3/B3/C4", false))
fstest.CheckListingWithPrecision(
t,
@@ -590,7 +591,7 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.Rmdirs(r.Fremote, "", false))
require.NoError(t, operations.Rmdirs(context.Background(), r.Fremote, "", false))
fstest.CheckListingWithPrecision(
t,
@@ -611,13 +612,13 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
func TestRmdirsLeaveRoot(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
r.ForceMkdir(r.Fremote)
r.ForceMkdir(context.Background(), r.Fremote)
require.NoError(t, operations.Mkdir(r.Fremote, "A1"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C1"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B1"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B1/C1"))
fstest.CheckListingWithPrecision(
t,
@@ -631,7 +632,7 @@ func TestRmdirsLeaveRoot(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.Rmdirs(r.Fremote, "A1", true))
require.NoError(t, operations.Rmdirs(context.Background(), r.Fremote, "A1", true))
fstest.CheckListingWithPrecision(
t,
@@ -653,7 +654,7 @@ func TestRcatSize(t *testing.T) {
file2 := r.WriteFile("potato2", body, t2)
// Test with known length
bodyReader := ioutil.NopCloser(strings.NewReader(body))
obj, err := operations.RcatSize(r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
obj, err := operations.RcatSize(context.Background(), r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
require.NoError(t, err)
assert.Equal(t, int64(len(body)), obj.Size())
assert.Equal(t, file1.Path, obj.Remote())
@@ -661,7 +662,7 @@ func TestRcatSize(t *testing.T) {
// Test with unknown length
bodyReader = ioutil.NopCloser(strings.NewReader(body)) // reset Reader
ioutil.NopCloser(strings.NewReader(body))
obj, err = operations.RcatSize(r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
obj, err = operations.RcatSize(context.Background(), r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
require.NoError(t, err)
assert.Equal(t, int64(len(body)), obj.Size())
assert.Equal(t, file2.Path, obj.Remote())
@@ -677,7 +678,7 @@ func TestCopyURL(t *testing.T) {
contents := "file contents\n"
file1 := r.WriteFile("file1", contents, t1)
file2 := r.WriteFile("file2", contents, t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fstest.CheckItems(t, r.Fremote)
// check when reading from regular HTTP server
@@ -688,7 +689,7 @@ func TestCopyURL(t *testing.T) {
ts := httptest.NewServer(handler)
defer ts.Close()
o, err := operations.CopyURL(r.Fremote, "file1", ts.URL)
o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL)
require.NoError(t, err)
assert.Equal(t, int64(len(contents)), o.Size())
@@ -704,7 +705,7 @@ func TestCopyURL(t *testing.T) {
tss := httptest.NewTLSServer(handler)
defer tss.Close()
o, err = operations.CopyURL(r.Fremote, "file2", tss.URL)
o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL)
require.NoError(t, err)
assert.Equal(t, int64(len(contents)), o.Size())
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
@@ -720,7 +721,7 @@ func TestMoveFile(t *testing.T) {
file2 := file1
file2.Path = "sub/file2"
err := operations.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err := operations.MoveFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2)
@@ -728,12 +729,12 @@ func TestMoveFile(t *testing.T) {
r.WriteFile("file1", "file1 contents", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err = operations.MoveFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2)
err = operations.MoveFile(r.Fremote, r.Fremote, file2.Path, file2.Path)
err = operations.MoveFile(context.Background(), r.Fremote, r.Fremote, file2.Path, file2.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2)
@@ -752,7 +753,7 @@ func TestCaseInsensitiveMoveFile(t *testing.T) {
file2 := file1
file2.Path = "sub/file2"
err := operations.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err := operations.MoveFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2)
@@ -760,7 +761,7 @@ func TestCaseInsensitiveMoveFile(t *testing.T) {
r.WriteFile("file1", "file1 contents", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.MoveFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err = operations.MoveFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2)
@@ -768,7 +769,7 @@ func TestCaseInsensitiveMoveFile(t *testing.T) {
file2Capitalized := file2
file2Capitalized.Path = "sub/File2"
err = operations.MoveFile(r.Fremote, r.Fremote, file2Capitalized.Path, file2.Path)
err = operations.MoveFile(context.Background(), r.Fremote, r.Fremote, file2Capitalized.Path, file2.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file2Capitalized)
@@ -787,10 +788,10 @@ func TestMoveFileBackupDir(t *testing.T) {
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
fstest.CheckItems(t, r.Flocal, file1)
file1old := r.WriteObject("dst/file1", "file1 contents old", t1)
file1old := r.WriteObject(context.Background(), "dst/file1", "file1 contents old", t1)
fstest.CheckItems(t, r.Fremote, file1old)
err := operations.MoveFile(r.Fremote, r.Flocal, file1.Path, file1.Path)
err := operations.MoveFile(context.Background(), r.Fremote, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
file1old.Path = "backup/dst/file1"
@@ -807,17 +808,17 @@ func TestCopyFile(t *testing.T) {
file2 := file1
file2.Path = "sub/file2"
err := operations.CopyFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
err = operations.CopyFile(r.Fremote, r.Flocal, file2.Path, file1.Path)
err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, file2.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
err = operations.CopyFile(r.Fremote, r.Fremote, file2.Path, file2.Path)
err = operations.CopyFile(context.Background(), r.Fremote, r.Fremote, file2.Path, file2.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
@@ -836,10 +837,10 @@ func TestCopyFileBackupDir(t *testing.T) {
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
fstest.CheckItems(t, r.Flocal, file1)
file1old := r.WriteObject("dst/file1", "file1 contents old", t1)
file1old := r.WriteObject(context.Background(), "dst/file1", "file1 contents old", t1)
fstest.CheckItems(t, r.Fremote, file1old)
err := operations.CopyFile(r.Fremote, r.Flocal, file1.Path, file1.Path)
err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
file1old.Path = "backup/dst/file1"
@@ -1135,19 +1136,19 @@ func TestDirMove(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
// Make some files and dirs
r.ForceMkdir(r.Fremote)
r.ForceMkdir(context.Background(), r.Fremote)
files := []fstest.Item{
r.WriteObject("A1/one", "one", t1),
r.WriteObject("A1/two", "two", t2),
r.WriteObject("A1/B1/three", "three", t3),
r.WriteObject("A1/B1/C1/four", "four", t1),
r.WriteObject("A1/B1/C2/five", "five", t2),
r.WriteObject(context.Background(), "A1/one", "one", t1),
r.WriteObject(context.Background(), "A1/two", "two", t2),
r.WriteObject(context.Background(), "A1/B1/three", "three", t3),
r.WriteObject(context.Background(), "A1/B1/C1/four", "four", t1),
r.WriteObject(context.Background(), "A1/B1/C2/five", "five", t2),
}
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B2"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "A1/B1/C3"))
fstest.CheckListingWithPrecision(
t,
@@ -1164,7 +1165,7 @@ func TestDirMove(t *testing.T) {
fs.GetModifyWindow(r.Fremote),
)
require.NoError(t, operations.DirMove(r.Fremote, "A1", "A2"))
require.NoError(t, operations.DirMove(context.Background(), r.Fremote, "A1", "A2"))
for i := range files {
files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
@@ -1194,7 +1195,7 @@ func TestDirMove(t *testing.T) {
features.DirMove = oldDirMove
}()
require.NoError(t, operations.DirMove(r.Fremote, "A2", "A3"))
require.NoError(t, operations.DirMove(context.Background(), r.Fremote, "A2", "A3"))
for i := range files {
files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)

51
fs/operations/rc.go
View File

@@ -1,6 +1,7 @@
package operations
import (
"context"
"strings"
"github.com/ncw/rclone/fs"
@@ -36,7 +37,7 @@ See the [lsjson command](/commands/rclone_lsjson/) for more information on the a
}
// List the directory
func rcList(in rc.Params) (out rc.Params, err error) {
func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
f, remote, err := rc.GetFsAndRemote(in)
if err != nil {
return nil, err
@@ -47,7 +48,7 @@ func rcList(in rc.Params) (out rc.Params, err error) {
return nil, err
}
var list = []*ListJSONItem{}
err = ListJSON(f, remote, &opt, func(item *ListJSONItem) error {
err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {
list = append(list, item)
return nil
})
@@ -77,7 +78,7 @@ See the [about command](/commands/rclone_size/) command for more information on
}
// About the remote
func rcAbout(in rc.Params) (out rc.Params, err error) {
func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) {
f, err := rc.GetFs(in)
if err != nil {
return nil, err
@@ -86,7 +87,7 @@ func rcAbout(in rc.Params) (out rc.Params, err error) {
if doAbout == nil {
return nil, errors.Errorf("%v doesn't support about", f)
}
u, err := doAbout()
u, err := doAbout(ctx)
if err != nil {
return nil, errors.Wrap(err, "about call failed")
}
@@ -107,8 +108,8 @@ func init() {
rc.Add(rc.Call{
Path: "operations/" + strings.ToLower(name) + "file",
AuthRequired: true,
Fn: func(in rc.Params) (rc.Params, error) {
return rcMoveOrCopyFile(in, copy)
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
return rcMoveOrCopyFile(ctx, in, copy)
},
Title: name + " a file from source remote to destination remote",
Help: `This takes the following parameters
@@ -123,7 +124,7 @@ func init() {
}
// Copy a file
func rcMoveOrCopyFile(in rc.Params, cp bool) (out rc.Params, err error) {
func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) {
srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(in, "srcFs", "srcRemote")
if err != nil {
return nil, err
@@ -132,7 +133,7 @@ func rcMoveOrCopyFile(in rc.Params, cp bool) (out rc.Params, err error) {
if err != nil {
return nil, err
}
return nil, moveOrCopyFile(dstFs, srcFs, dstRemote, srcRemote, cp)
return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp)
}
func init() {
@@ -159,8 +160,8 @@ func init() {
rc.Add(rc.Call{
Path: "operations/" + op.name,
AuthRequired: true,
Fn: func(in rc.Params) (rc.Params, error) {
return rcSingleCommand(in, op.name, op.noRemote)
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
return rcSingleCommand(ctx, in, op.name, op.noRemote)
},
Title: op.title,
Help: `This takes the following parameters
@@ -174,7 +175,7 @@ See the [` + op.name + ` command](/commands/rclone_` + op.name + `/) command for
}
// Run a single command, eg Mkdir
func rcSingleCommand(in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
var (
f fs.Fs
remote string
@@ -189,34 +190,34 @@ func rcSingleCommand(in rc.Params, name string, noRemote bool) (out rc.Params, e
}
switch name {
case "mkdir":
return nil, Mkdir(f, remote)
return nil, Mkdir(ctx, f, remote)
case "rmdir":
return nil, Rmdir(f, remote)
return nil, Rmdir(ctx, f, remote)
case "purge":
return nil, Purge(f, remote)
return nil, Purge(ctx, f, remote)
case "rmdirs":
leaveRoot, err := in.GetBool("leaveRoot")
if rc.NotErrParamNotFound(err) {
return nil, err
}
return nil, Rmdirs(f, remote, leaveRoot)
return nil, Rmdirs(ctx, f, remote, leaveRoot)
case "delete":
return nil, Delete(f)
return nil, Delete(ctx, f)
case "deletefile":
o, err := f.NewObject(remote)
o, err := f.NewObject(ctx, remote)
if err != nil {
return nil, err
}
return nil, DeleteFile(o)
return nil, DeleteFile(ctx, o)
case "copyurl":
url, err := in.GetString("url")
if err != nil {
return nil, err
}
_, err = CopyURL(f, remote, url)
_, err = CopyURL(ctx, f, remote, url)
return nil, err
case "cleanup":
return nil, CleanUp(f)
return nil, CleanUp(ctx, f)
}
panic("unknown rcSingleCommand type")
}
@@ -242,12 +243,12 @@ See the [size command](/commands/rclone_size/) command for more information on t
}
// Size a directory
func rcSize(in rc.Params) (out rc.Params, err error) {
func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
f, err := rc.GetFs(in)
if err != nil {
return nil, err
}
count, bytes, err := Count(f)
count, bytes, err := Count(ctx, f)
if err != nil {
return nil, err
}
@@ -278,12 +279,12 @@ See the [link command](/commands/rclone_link/) command for more information on t
}
// Make a public link
func rcPublicLink(in rc.Params) (out rc.Params, err error) {
func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) {
f, remote, err := rc.GetFsAndRemote(in)
if err != nil {
return nil, err
}
url, err := PublicLink(f, remote)
url, err := PublicLink(ctx, f, remote)
if err != nil {
return nil, err
}
@@ -357,7 +358,7 @@ This command does not have a command line equivalent so use this instead:
}
// Fsinfo the remote
func rcFsInfo(in rc.Params) (out rc.Params, err error) {
func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) {
f, err := rc.GetFs(in)
if err != nil {
return nil, err

81
fs/operations/rc_test.go
View File

@@ -1,6 +1,7 @@
package operations_test
import (
"context"
"net/http"
"net/http/httptest"
"testing"
@@ -31,7 +32,7 @@ func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
func TestRcAbout(t *testing.T) {
r, call := rcNewRun(t, "operations/about")
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
// Will get an error if remote doesn't support About
expectedErr := r.Fremote.Features().About == nil
@@ -39,7 +40,7 @@ func TestRcAbout(t *testing.T) {
in := rc.Params{
"fs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
if expectedErr {
assert.Error(t, err)
return
@@ -58,7 +59,7 @@ func TestRcCleanup(t *testing.T) {
in := rc.Params{
"fs": r.LocalName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.Error(t, err)
assert.Equal(t, rc.Params(nil), out)
assert.Contains(t, err.Error(), "doesn't support cleanup")
@@ -69,7 +70,7 @@ func TestRcCopyfile(t *testing.T) {
r, call := rcNewRun(t, "operations/copyfile")
defer r.Finalise()
file1 := r.WriteFile("file1", "file1 contents", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote)
@@ -79,7 +80,7 @@ func TestRcCopyfile(t *testing.T) {
"dstFs": r.FremoteName,
"dstRemote": "file1-renamed",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -94,7 +95,7 @@ func TestRcCopyurl(t *testing.T) {
defer r.Finalise()
contents := "file1 contents\n"
file1 := r.WriteFile("file1", contents, t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fstest.CheckItems(t, r.Fremote)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -108,7 +109,7 @@ func TestRcCopyurl(t *testing.T) {
"remote": "file1",
"url": ts.URL,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -120,15 +121,15 @@ func TestRcDelete(t *testing.T) {
r, call := rcNewRun(t, "operations/delete")
defer r.Finalise()
file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject("large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(context.Background(), "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
in := rc.Params{
"fs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -140,15 +141,15 @@ func TestRcDeletefile(t *testing.T) {
r, call := rcNewRun(t, "operations/deletefile")
defer r.Finalise()
file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject("medium", "------------------------------------------------------------", t1) // 60 bytes
file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
fstest.CheckItems(t, r.Fremote, file1, file2)
in := rc.Params{
"fs": r.FremoteName,
"remote": "small",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -160,8 +161,8 @@ func TestRcList(t *testing.T) {
r, call := rcNewRun(t, "operations/list")
defer r.Finalise()
file1 := r.WriteObject("a", "a", t1)
file2 := r.WriteObject("subdir/b", "bb", t2)
file1 := r.WriteObject(context.Background(), "a", "a", t1)
file2 := r.WriteObject(context.Background(), "subdir/b", "bb", t2)
fstest.CheckItems(t, r.Fremote, file1, file2)
@@ -169,7 +170,7 @@ func TestRcList(t *testing.T) {
"fs": r.FremoteName,
"remote": "",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
list := out["list"].([]*operations.ListJSONItem)
@@ -201,7 +202,7 @@ func TestRcList(t *testing.T) {
"recurse": true,
},
}
out, err = call.Fn(in)
out, err = call.Fn(context.Background(), in)
require.NoError(t, err)
list = out["list"].([]*operations.ListJSONItem)
@@ -224,7 +225,7 @@ func TestRcList(t *testing.T) {
func TestRcMkdir(t *testing.T) {
r, call := rcNewRun(t, "operations/mkdir")
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
@@ -232,7 +233,7 @@ func TestRcMkdir(t *testing.T) {
"fs": r.FremoteName,
"remote": "subdir",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -244,7 +245,7 @@ func TestRcMovefile(t *testing.T) {
r, call := rcNewRun(t, "operations/movefile")
defer r.Finalise()
file1 := r.WriteFile("file1", "file1 contents", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote)
@@ -254,7 +255,7 @@ func TestRcMovefile(t *testing.T) {
"dstFs": r.FremoteName,
"dstRemote": "file1-renamed",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -267,7 +268,7 @@ func TestRcMovefile(t *testing.T) {
func TestRcPurge(t *testing.T) {
r, call := rcNewRun(t, "operations/purge")
defer r.Finalise()
file1 := r.WriteObject("subdir/file1", "subdir/file1 contents", t1)
file1 := r.WriteObject(context.Background(), "subdir/file1", "subdir/file1 contents", t1)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
@@ -275,7 +276,7 @@ func TestRcPurge(t *testing.T) {
"fs": r.FremoteName,
"remote": "subdir",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -286,8 +287,8 @@ func TestRcPurge(t *testing.T) {
func TestRcRmdir(t *testing.T) {
r, call := rcNewRun(t, "operations/rmdir")
defer r.Finalise()
r.Mkdir(r.Fremote)
assert.NoError(t, r.Fremote.Mkdir("subdir"))
r.Mkdir(context.Background(), r.Fremote)
assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(r.Fremote))
@@ -295,7 +296,7 @@ func TestRcRmdir(t *testing.T) {
"fs": r.FremoteName,
"remote": "subdir",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -306,9 +307,9 @@ func TestRcRmdir(t *testing.T) {
func TestRcRmdirs(t *testing.T) {
r, call := rcNewRun(t, "operations/rmdirs")
defer r.Finalise()
r.Mkdir(r.Fremote)
assert.NoError(t, r.Fremote.Mkdir("subdir"))
assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir"))
r.Mkdir(context.Background(), r.Fremote)
assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir"))
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(r.Fremote))
@@ -316,21 +317,21 @@ func TestRcRmdirs(t *testing.T) {
"fs": r.FremoteName,
"remote": "subdir",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(r.Fremote))
assert.NoError(t, r.Fremote.Mkdir("subdir"))
assert.NoError(t, r.Fremote.Mkdir("subdir/subsubdir"))
assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir"))
in = rc.Params{
"fs": r.FremoteName,
"remote": "subdir",
"leaveRoot": true,
}
out, err = call.Fn(in)
out, err = call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -342,15 +343,15 @@ func TestRcRmdirs(t *testing.T) {
func TestRcSize(t *testing.T) {
r, call := rcNewRun(t, "operations/size")
defer r.Finalise()
file1 := r.WriteObject("small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject("subdir/medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject("subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes
file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(context.Background(), "subdir/medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(context.Background(), "subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
in := rc.Params{
"fs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"count": int64(3),
@@ -366,7 +367,7 @@ func TestRcPublicLink(t *testing.T) {
"fs": r.FremoteName,
"remote": "",
}
_, err := call.Fn(in)
_, err := call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "doesn't support public links")
}
@@ -378,7 +379,7 @@ func TestRcFsInfo(t *testing.T) {
in := rc.Params{
"fs": r.FremoteName,
}
got, err := call.Fn(in)
got, err := call.Fn(context.Background(), in)
require.NoError(t, err)
want := operations.GetFsInfo(r.Fremote)
assert.Equal(t, want.Name, got["Name"])

7
fs/operations/reopen.go
View File

@@ -1,6 +1,7 @@
package operations
import (
"context"
"io"
"sync"
@@ -10,6 +11,7 @@ import (
// reOpen is a wrapper for an object reader which reopens the stream on error
type reOpen struct {
ctx context.Context
mu sync.Mutex // mutex to protect the below
src fs.Object // object to open
hashOption *fs.HashesOption // option to pass to initial open
@@ -33,8 +35,9 @@ var (
//
// If rangeOption is set then this will applied when reading from the
// start, and updated on retries.
func newReOpen(src fs.Object, hashOption *fs.HashesOption, rangeOption *fs.RangeOption, maxTries int) (rc io.ReadCloser, err error) {
func newReOpen(ctx context.Context, src fs.Object, hashOption *fs.HashesOption, rangeOption *fs.RangeOption, maxTries int) (rc io.ReadCloser, err error) {
h := &reOpen{
ctx: ctx,
src: src,
hashOption: hashOption,
rangeOption: rangeOption,
@@ -76,7 +79,7 @@ func (h *reOpen) open() error {
if h.tries > h.maxTries {
h.err = errorTooManyTries
} else {
h.rc, h.err = h.src.Open(opts...)
h.rc, h.err = h.src.Open(h.ctx, opts...)
}
if h.err != nil {
if h.tries > 1 {

7
fs/operations/reopen_test.go
View File

@@ -1,6 +1,7 @@
package operations
import (
"context"
"io"
"io/ioutil"
"testing"
@@ -29,8 +30,8 @@ type reOpenTestObject struct {
// Open opens the file for read. Call Close() on the returned io.ReadCloser
//
// This will break after reading the number of bytes in breaks
func (o *reOpenTestObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
rc, err := o.Object.Open(options...)
func (o *reOpenTestObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
rc, err := o.Object.Open(ctx, options...)
if err != nil {
return nil, err
}
@@ -82,7 +83,7 @@ func TestReOpen(t *testing.T) {
breaks: breaks,
}
hashOption := &fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}
return newReOpen(src, hashOption, rangeOption, maxRetries)
return newReOpen(context.Background(), src, hashOption, rangeOption, maxRetries)
}
t.Run("Basics", func(t *testing.T) {

8
fs/rc/config.go
View File

@@ -5,6 +5,8 @@
package rc
import (
"context"
"github.com/pkg/errors"
)
@@ -36,7 +38,7 @@ func init() {
}
// Show the list of all the option blocks
func rcOptionsBlocks(in Params) (out Params, err error) {
func rcOptionsBlocks(ctx context.Context, in Params) (out Params, err error) {
options := []string{}
for name := range optionBlock {
options = append(options, name)
@@ -61,7 +63,7 @@ map to the external options very easily with a few exceptions.
}
// Show the list of all the option blocks
func rcOptionsGet(in Params) (out Params, err error) {
func rcOptionsGet(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
for name, options := range optionBlock {
out[name] = options
@@ -103,7 +105,7 @@ And this sets NOTICE level logs (normal without -v)
}
// Set an option in an option block
func rcOptionsSet(in Params) (out Params, err error) {
func rcOptionsSet(ctx context.Context, in Params) (out Params, err error) {
for name, options := range in {
current := optionBlock[name]
if current == nil {

13
fs/rc/config_test.go
View File

@@ -1,6 +1,7 @@
package rc
import (
"context"
"fmt"
"testing"
@@ -47,7 +48,7 @@ func TestOptionsBlocks(t *testing.T) {
call := Calls.Get("options/blocks")
require.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, Params{"options": []string{"potato"}}, out)
@@ -59,7 +60,7 @@ func TestOptionsGet(t *testing.T) {
call := Calls.Get("options/get")
require.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, Params{"potato": &testOptions}, out)
@@ -83,7 +84,7 @@ func TestOptionsSet(t *testing.T) {
"Int": 50,
},
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.Nil(t, out)
assert.Equal(t, 50, testOptions.Int)
@@ -91,7 +92,7 @@ func TestOptionsSet(t *testing.T) {
assert.Equal(t, 1, reloaded)
// error from reload
_, err = call.Fn(in)
_, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "error while reloading")
@@ -101,7 +102,7 @@ func TestOptionsSet(t *testing.T) {
"Int": 50,
},
}
_, err = call.Fn(in)
_, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown option block")
@@ -109,7 +110,7 @@ func TestOptionsSet(t *testing.T) {
in = Params{
"potato": []string{"a", "b"},
}
_, err = call.Fn(in)
_, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to write options")

17
fs/rc/internal.go
View File

@@ -3,6 +3,7 @@
package rc
import (
"context"
"os"
"runtime"
@@ -35,7 +36,7 @@ check that parameter passing is working properly.`,
}
// Echo the input to the output parameters
func rcNoop(in Params) (out Params, err error) {
func rcNoop(ctx context.Context, in Params) (out Params, err error) {
return in, nil
}
@@ -51,7 +52,7 @@ Useful for testing error handling.`,
}
// Return an error regardless
func rcError(in Params) (out Params, err error) {
func rcError(ctx context.Context, in Params) (out Params, err error) {
return nil, errors.Errorf("arbitrary error on input %+v", in)
}
@@ -67,7 +68,7 @@ the commands response.`,
}
// List the registered commands
func rcList(in Params) (out Params, err error) {
func rcList(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
out["commands"] = Calls.List()
return out, nil
@@ -85,7 +86,7 @@ Useful for stopping rclone process.`,
}
// Return PID of current process
func rcPid(in Params) (out Params, err error) {
func rcPid(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
out["pid"] = os.Getpid()
return out, nil
@@ -111,7 +112,7 @@ The most interesting values for most people are:
}
// Return the memory statistics
func rcMemStats(in Params) (out Params, err error) {
func rcMemStats(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
var m runtime.MemStats
runtime.ReadMemStats(&m)
@@ -152,7 +153,7 @@ memory problems.
}
// Do a garbage collection run
func rcGc(in Params) (out Params, err error) {
func rcGc(ctx context.Context, in Params) (out Params, err error) {
runtime.GC()
return nil, nil
}
@@ -177,7 +178,7 @@ This shows the current version of go and the go runtime
}
// Return version info
func rcVersion(in Params) (out Params, err error) {
func rcVersion(ctx context.Context, in Params) (out Params, err error) {
decomposed, err := version.New(fs.Version)
if err != nil {
return nil, err
@@ -209,7 +210,7 @@ Returns
}
// Return obscured string
func rcObscure(in Params) (out Params, err error) {
func rcObscure(ctx context.Context, in Params) (out Params, err error) {
clear, err := in.GetString("clear")
if err != nil {
return nil, err

17
fs/rc/internal_test.go
View File

@@ -1,6 +1,7 @@
package rc
import (
"context"
"runtime"
"testing"
@@ -18,7 +19,7 @@ func TestInternalNoop(t *testing.T) {
"String": "hello",
"Int": 42,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, in, out)
@@ -28,7 +29,7 @@ func TestInternalError(t *testing.T) {
call := Calls.Get("rc/error")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.Error(t, err)
require.Nil(t, out)
}
@@ -37,7 +38,7 @@ func TestInternalList(t *testing.T) {
call := Calls.Get("rc/list")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, Params{"commands": Calls.List()}, out)
@@ -47,7 +48,7 @@ func TestCorePid(t *testing.T) {
call := Calls.Get("core/pid")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
pid := out["pid"]
@@ -60,7 +61,7 @@ func TestCoreMemstats(t *testing.T) {
call := Calls.Get("core/memstats")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
sys := out["Sys"]
@@ -73,7 +74,7 @@ func TestCoreGC(t *testing.T) {
call := Calls.Get("core/gc")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.Nil(t, out)
assert.Equal(t, Params(nil), out)
@@ -83,7 +84,7 @@ func TestCoreVersion(t *testing.T) {
call := Calls.Get("core/version")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, fs.Version, out["version"])
@@ -101,7 +102,7 @@ func TestCoreObscure(t *testing.T) {
in := Params{
"clear": "potato",
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, in["clear"], obscure.MustReveal(out["obscured"].(string)))

26
fs/rc/job.go
View File

@@ -3,6 +3,7 @@
package rc
import (
"context"
"sync"
"sync/atomic"
"time"
@@ -14,14 +15,15 @@ import (
// Job describes a asynchronous task started via the rc package
type Job struct {
mu sync.Mutex
ID int64 `json:"id"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Error string `json:"error"`
Finished bool `json:"finished"`
Success bool `json:"success"`
Duration float64 `json:"duration"`
Output Params `json:"output"`
ID int64 `json:"id"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Error string `json:"error"`
Finished bool `json:"finished"`
Success bool `json:"success"`
Duration float64 `json:"duration"`
Output Params `json:"output"`
Context context.Context `json:"-"`
}
// Jobs describes a collection of running tasks
@@ -121,7 +123,7 @@ func (job *Job) run(fn Func, in Params) {
job.finish(nil, errors.Errorf("panic received: %v", r))
}
}()
job.finish(fn(in))
job.finish(fn(job.Context, in))
}
// NewJob start a new Job off
@@ -129,6 +131,7 @@ func (jobs *Jobs) NewJob(fn Func, in Params) *Job {
job := &Job{
ID: atomic.AddInt64(&jobID, 1),
StartTime: time.Now(),
Context: context.Background(),
}
go job.run(fn, in)
jobs.mu.Lock()
@@ -164,12 +167,13 @@ Results
- startTime - time the job started (eg "2018-10-26T18:50:20.528336039+01:00")
- success - boolean - true for success false otherwise
- output - output of the job as would have been returned if called synchronously
- progress - output of the progress related to the underlying job
`,
})
}
// Returns the status of a job
func rcJobStatus(in Params) (out Params, err error) {
func rcJobStatus(ctx context.Context, in Params) (out Params, err error) {
jobID, err := in.GetInt64("jobid")
if err != nil {
return nil, err
@@ -202,7 +206,7 @@ Results
}
// Returns the status of a job
func rcJobList(in Params) (out Params, err error) {
func rcJobList(ctx context.Context, in Params) (out Params, err error) {
out = make(Params)
out["jobids"] = running.IDs()
return out, nil

18
fs/rc/job_test.go
View File

@@ -1,6 +1,7 @@
package rc
import (
"context"
"runtime"
"testing"
"time"
@@ -35,7 +36,7 @@ func TestJobsExpire(t *testing.T) {
jobs := newJobs()
jobs.expireInterval = time.Millisecond
assert.Equal(t, false, jobs.expireRunning)
job := jobs.NewJob(func(in Params) (Params, error) {
job := jobs.NewJob(func(ctx context.Context, in Params) (Params, error) {
defer close(wait)
return in, nil
}, Params{})
@@ -56,7 +57,7 @@ func TestJobsExpire(t *testing.T) {
jobs.mu.Unlock()
}
var noopFn = func(in Params) (Params, error) {
var noopFn = func(ctx context.Context, in Params) (Params, error) {
return nil, nil
}
@@ -80,7 +81,8 @@ func TestJobsGet(t *testing.T) {
assert.Nil(t, jobs.Get(123123123123))
}
var longFn = func(in Params) (Params, error) {
var longFn = func(ctx context.Context, in Params) (Params, error) {
// TODO get execution time from context?
time.Sleep(1 * time.Hour)
return nil, nil
}
@@ -144,7 +146,7 @@ func TestJobFinish(t *testing.T) {
// part of NewJob, now just test the panic catching
func TestJobRunPanic(t *testing.T) {
wait := make(chan struct{})
boom := func(in Params) (Params, error) {
boom := func(ctx context.Context, in Params) (Params, error) {
sleepJob()
defer close(wait)
panic("boom")
@@ -200,7 +202,7 @@ func TestRcJobStatus(t *testing.T) {
call := Calls.Get("job/status")
assert.NotNil(t, call)
in := Params{"jobid": 1}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, float64(1), out["id"])
@@ -209,12 +211,12 @@ func TestRcJobStatus(t *testing.T) {
assert.Equal(t, false, out["success"])
in = Params{"jobid": 123123123}
_, err = call.Fn(in)
_, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "job not found")
in = Params{"jobidx": 123123123}
_, err = call.Fn(in)
_, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Contains(t, err.Error(), "Didn't find key")
}
@@ -227,7 +229,7 @@ func TestRcJobList(t *testing.T) {
call := Calls.Get("job/list")
assert.NotNil(t, call)
in := Params{}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
require.NotNil(t, out)
assert.Equal(t, Params{"jobids": []int64{1}}, out)

6
fs/rc/rcserver/rcserver.go
View File

@@ -187,7 +187,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
if isAsync {
out, err = rc.StartJob(call.Fn, in)
} else {
out, err = call.Fn(in)
out, err = call.Fn(r.Context(), in)
}
if err != nil {
writeError(path, in, w, err, http.StatusInternalServerError)
@@ -230,7 +230,7 @@ func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string
}
if path == "" || strings.HasSuffix(path, "/") {
path = strings.Trim(path, "/")
entries, err := list.DirSorted(f, false, path)
entries, err := list.DirSorted(r.Context(), f, false, path)
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to list directory"), http.StatusInternalServerError)
return
@@ -244,7 +244,7 @@ func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string
directory.Serve(w, r)
} else {
path = strings.Trim(path, "/")
o, err := f.NewObject(path)
o, err := f.NewObject(r.Context(), path)
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to find object"), http.StatusInternalServerError)
return

3
fs/rc/registry.go
View File

@@ -3,6 +3,7 @@
package rc
import (
"context"
"sort"
"strings"
"sync"
@@ -11,7 +12,7 @@ import (
)
// Func defines a type for a remote control function
type Func func(in Params) (out Params, err error)
type Func func(ctx context.Context, in Params) (out Params, err error)
// Call defines info about a remote control function and is used in
// the Add function to create new entry points.

14
fs/sync/rc.go
View File

@@ -1,6 +1,8 @@
package sync
import (
"context"
"github.com/ncw/rclone/fs/rc"
)
@@ -14,8 +16,8 @@ func init() {
rc.Add(rc.Call{
Path: "sync/" + name,
AuthRequired: true,
Fn: func(in rc.Params) (rc.Params, error) {
return rcSyncCopyMove(in, name)
Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
return rcSyncCopyMove(ctx, in, name)
},
Title: name + " a directory from source remote to destination remote",
Help: `This takes the following parameters
@@ -30,7 +32,7 @@ See the [` + name + ` command](/commands/rclone_` + name + `/) command for more
}
// Sync/Copy/Move a file
func rcSyncCopyMove(in rc.Params, name string) (out rc.Params, err error) {
func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Params, err error) {
srcFs, err := rc.GetFsNamed(in, "srcFs")
if err != nil {
return nil, err
@@ -45,15 +47,15 @@ func rcSyncCopyMove(in rc.Params, name string) (out rc.Params, err error) {
}
switch name {
case "sync":
return nil, Sync(dstFs, srcFs, createEmptySrcDirs)
return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)
case "copy":
return nil, CopyDir(dstFs, srcFs, createEmptySrcDirs)
return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)
case "move":
deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs")
if rc.NotErrParamNotFound(err) {
return nil, err
}
return nil, MoveDir(dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
}
panic("unknown rcSyncCopyMove type")
}

25
fs/sync/rc_test.go
View File

@@ -1,6 +1,7 @@
package sync
import (
"context"
"testing"
"github.com/ncw/rclone/fs/cache"
@@ -26,11 +27,11 @@ func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
func TestRcCopy(t *testing.T) {
r, call := rcNewRun(t, "sync/copy")
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
file1 := r.WriteBoth("file1", "file1 contents", t1)
file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
file2 := r.WriteFile("subdir/file2", "file2 contents", t2)
file3 := r.WriteObject("subdir/subsubdir/file3", "file3 contents", t3)
file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1, file3)
@@ -39,7 +40,7 @@ func TestRcCopy(t *testing.T) {
"srcFs": r.LocalName,
"dstFs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -51,11 +52,11 @@ func TestRcCopy(t *testing.T) {
func TestRcMove(t *testing.T) {
r, call := rcNewRun(t, "sync/move")
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
file1 := r.WriteBoth("file1", "file1 contents", t1)
file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
file2 := r.WriteFile("subdir/file2", "file2 contents", t2)
file3 := r.WriteObject("subdir/subsubdir/file3", "file3 contents", t3)
file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1, file3)
@@ -64,7 +65,7 @@ func TestRcMove(t *testing.T) {
"srcFs": r.LocalName,
"dstFs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
@@ -76,11 +77,11 @@ func TestRcMove(t *testing.T) {
func TestRcSync(t *testing.T) {
r, call := rcNewRun(t, "sync/sync")
defer r.Finalise()
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
file1 := r.WriteBoth("file1", "file1 contents", t1)
file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
file2 := r.WriteFile("subdir/file2", "file2 contents", t2)
file3 := r.WriteObject("subdir/subsubdir/file3", "file3 contents", t3)
file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1, file3)
@@ -89,7 +90,7 @@ func TestRcSync(t *testing.T) {
"srcFs": r.LocalName,
"dstFs": r.FremoteName,
}
out, err := call.Fn(in)
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)

70
fs/sync/sync.go
View File

@@ -65,7 +65,7 @@ type syncCopyMove struct {
suffix string // suffix to add to files placed in backupDir
}
func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
if (deleteMode != fs.DeleteModeOff || DoMove) && operations.Overlapping(fdst, fsrc) {
return nil, fserrors.FatalError(fs.ErrorOverlapping)
}
@@ -91,7 +91,7 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
toBeRenamed: newPipe(accounting.Stats.SetRenameQueue, fs.Config.MaxBacklog),
trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
}
s.ctx, s.cancel = context.WithCancel(context.Background())
s.ctx, s.cancel = context.WithCancel(ctx)
if s.noTraverse && s.deleteMode != fs.DeleteModeOff {
fs.Errorf(nil, "Ignoring --no-traverse with sync")
s.noTraverse = false
@@ -219,7 +219,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
accounting.Stats.Checking(src.Remote())
// Check to see if can store this
if src.Storable() {
if operations.NeedTransfer(pair.Dst, pair.Src) {
if operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
// If files are treated as immutable, fail if destination exists and does not match
if fs.Config.Immutable && pair.Dst != nil {
fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
@@ -228,8 +228,8 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
// If destination already exists, then we must move it into --backup-dir if required
if pair.Dst != nil && s.backupDir != nil {
remoteWithSuffix := operations.SuffixName(pair.Dst.Remote())
overwritten, _ := s.backupDir.NewObject(remoteWithSuffix)
_, err := operations.Move(s.backupDir, overwritten, remoteWithSuffix, pair.Dst)
overwritten, _ := s.backupDir.NewObject(s.ctx, remoteWithSuffix)
_, err := operations.Move(s.ctx, s.backupDir, overwritten, remoteWithSuffix, pair.Dst)
if err != nil {
s.processError(err)
} else {
@@ -251,7 +251,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
// If moving need to delete the files we don't need to copy
if s.DoMove {
// Delete src if no error on copy
s.processError(operations.DeleteFile(src))
s.processError(operations.DeleteFile(s.ctx, src))
}
}
}
@@ -280,7 +280,7 @@ func (s *syncCopyMove) pairRenamer(in *pipe, out *pipe, wg *sync.WaitGroup) {
}
// pairCopyOrMove reads Objects on in and moves or copies them.
func (s *syncCopyMove) pairCopyOrMove(in *pipe, fdst fs.Fs, wg *sync.WaitGroup) {
func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs, wg *sync.WaitGroup) {
defer wg.Done()
var err error
for {
@@ -290,9 +290,9 @@ func (s *syncCopyMove) pairCopyOrMove(in *pipe, fdst fs.Fs, wg *sync.WaitGroup)
}
src := pair.Src
if s.DoMove {
_, err = operations.Move(fdst, pair.Dst, src.Remote(), src)
_, err = operations.Move(ctx, fdst, pair.Dst, src.Remote(), src)
} else {
_, err = operations.Copy(fdst, pair.Dst, src.Remote(), src)
_, err = operations.Copy(ctx, fdst, pair.Dst, src.Remote(), src)
}
s.processError(err)
}
@@ -317,7 +317,7 @@ func (s *syncCopyMove) stopCheckers() {
func (s *syncCopyMove) startTransfers() {
s.transfersWg.Add(fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
go s.pairCopyOrMove(s.toBeUploaded, s.fdst, &s.transfersWg)
go s.pairCopyOrMove(s.ctx, s.toBeUploaded, s.fdst, &s.transfersWg)
}
}
@@ -380,7 +380,7 @@ func (s *syncCopyMove) startDeleters() {
s.deletersWg.Add(1)
go func() {
defer s.deletersWg.Done()
err := operations.DeleteFilesWithBackupDir(s.deleteFilesCh, s.backupDir)
err := operations.DeleteFilesWithBackupDir(s.ctx, s.deleteFilesCh, s.backupDir)
s.processError(err)
}()
}
@@ -427,12 +427,12 @@ func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
}
close(toDelete)
}()
return operations.DeleteFilesWithBackupDir(toDelete, s.backupDir)
return operations.DeleteFilesWithBackupDir(s.ctx, toDelete, s.backupDir)
}
// This deletes the empty directories in the slice passed in. It
// ignores any errors deleting directories
func deleteEmptyDirectories(f fs.Fs, entriesMap map[string]fs.DirEntry) error {
func deleteEmptyDirectories(ctx context.Context, f fs.Fs, entriesMap map[string]fs.DirEntry) error {
if len(entriesMap) == 0 {
return nil
}
@@ -454,7 +454,7 @@ func deleteEmptyDirectories(f fs.Fs, entriesMap map[string]fs.DirEntry) error {
dir, ok := entry.(fs.Directory)
if ok {
// TryRmdir only deletes empty directories
err := operations.TryRmdir(f, dir.Remote())
err := operations.TryRmdir(ctx, f, dir.Remote())
if err != nil {
fs.Debugf(fs.LogDirName(f, dir.Remote()), "Failed to Rmdir: %v", err)
errorCount++
@@ -476,7 +476,7 @@ func deleteEmptyDirectories(f fs.Fs, entriesMap map[string]fs.DirEntry) error {
// This copies the empty directories in the slice passed in and logs
// any errors copying the directories
func copyEmptyDirectories(f fs.Fs, entries map[string]fs.DirEntry) error {
func copyEmptyDirectories(ctx context.Context, f fs.Fs, entries map[string]fs.DirEntry) error {
if len(entries) == 0 {
return nil
}
@@ -485,7 +485,7 @@ func copyEmptyDirectories(f fs.Fs, entries map[string]fs.DirEntry) error {
for _, entry := range entries {
dir, ok := entry.(fs.Directory)
if ok {
err := operations.Mkdir(f, dir.Remote())
err := operations.Mkdir(ctx, f, dir.Remote())
if err != nil {
fs.Errorf(fs.LogDirName(f, dir.Remote()), "Failed to Mkdir: %v", err)
} else {
@@ -526,7 +526,7 @@ func (s *syncCopyMove) srcParentDirCheck(entry fs.DirEntry) {
// it may return an empty string in which case no hash could be made
func (s *syncCopyMove) renameHash(obj fs.Object) (hash string) {
var err error
hash, err = obj.Hash(s.commonHash)
hash, err = obj.Hash(s.ctx, s.commonHash)
if err != nil {
fs.Debugf(obj, "Hash failed: %v", err)
return ""
@@ -616,10 +616,10 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool {
}
// Find dst object we are about to overwrite if it exists
dstOverwritten, _ := s.fdst.NewObject(src.Remote())
dstOverwritten, _ := s.fdst.NewObject(s.ctx, src.Remote())
// Rename dst to have name src.Remote()
_, err := operations.Move(s.fdst, dstOverwritten, src.Remote(), dst)
_, err := operations.Move(s.ctx, s.fdst, dstOverwritten, src.Remote(), dst)
if err != nil {
fs.Debugf(src, "Failed to rename to %q: %v", dst.Remote(), err)
return false
@@ -688,7 +688,7 @@ func (s *syncCopyMove) run() error {
s.stopDeleters()
if s.copyEmptySrcDirs {
s.processError(copyEmptyDirectories(s.fdst, s.srcEmptyDirs))
s.processError(copyEmptyDirectories(s.ctx, s.fdst, s.srcEmptyDirs))
}
// Delete files after
@@ -705,7 +705,7 @@ func (s *syncCopyMove) run() error {
if s.currentError() != nil && !fs.Config.IgnoreErrors {
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeletingDirs)
} else {
s.processError(deleteEmptyDirectories(s.fdst, s.dstEmptyDirs))
s.processError(deleteEmptyDirectories(s.ctx, s.fdst, s.dstEmptyDirs))
}
}
@@ -713,7 +713,7 @@ func (s *syncCopyMove) run() error {
// if DoMove and --delete-empty-src-dirs flag is set
if s.DoMove && s.deleteEmptySrcDirs {
//delete empty subdirectories that were part of the move
s.processError(deleteEmptyDirectories(s.fsrc, s.srcEmptyDirs))
s.processError(deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
}
// cancel the context to free resources
@@ -802,7 +802,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
}
// Match is called when src and dst are present, so sync src to dst
func (s *syncCopyMove) Match(dst, src fs.DirEntry) (recurse bool) {
func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
switch srcX := src.(type) {
case fs.Object:
s.srcEmptyDirsMu.Lock()
@@ -852,7 +852,7 @@ func (s *syncCopyMove) Match(dst, src fs.DirEntry) (recurse bool) {
// If DoMove is true then files will be moved instead of copied
//
// dir is the start directory, "" for root
func runSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
if deleteMode != fs.DeleteModeOff && DoMove {
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
}
@@ -862,7 +862,7 @@ func runSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))
}
// only delete stuff during in this pass
do, err := newSyncCopyMove(fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs)
do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs)
if err != nil {
return err
}
@@ -873,7 +873,7 @@ func runSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
// Next pass does a copy only
deleteMode = fs.DeleteModeOff
}
do, err := newSyncCopyMove(fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs)
do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs)
if err != nil {
return err
}
@@ -881,22 +881,22 @@ func runSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de
}
// Sync fsrc into fdst
func Sync(fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
return runSyncCopyMove(fdst, fsrc, fs.Config.DeleteMode, false, false, copyEmptySrcDirs)
func Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
return runSyncCopyMove(ctx, fdst, fsrc, fs.Config.DeleteMode, false, false, copyEmptySrcDirs)
}
// CopyDir copies fsrc into fdst
func CopyDir(fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
return runSyncCopyMove(fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs)
func CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs)
}
// moveDir moves fsrc into fdst
func moveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
return runSyncCopyMove(fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)
func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)
}
// MoveDir moves fsrc into fdst
func MoveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
if operations.Same(fdst, fsrc) {
fs.Errorf(fdst, "Nothing to do as source and destination are the same")
return nil
@@ -909,7 +909,7 @@ func MoveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) e
return nil
}
fs.Debugf(fdst, "Using server side directory move")
err := fdstDirMove(fsrc, "", "")
err := fdstDirMove(ctx, fsrc, "", "")
switch err {
case fs.ErrorCantDirMove, fs.ErrorDirExists:
fs.Infof(fdst, "Server side directory move failed - fallback to file moves: %v", err)
@@ -924,5 +924,5 @@ func MoveDir(fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) e
}
// Otherwise move the files one by one
return moveDir(fdst, fsrc, deleteEmptySrcDirs, copyEmptySrcDirs)
return moveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, copyEmptySrcDirs)
}

241
fs/sync/sync_test.go
View File

@@ -3,6 +3,7 @@
package sync
import (
"context"
"runtime"
"testing"
"time"
@@ -38,10 +39,10 @@ func TestCopyWithDryRun(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fs.Config.DryRun = true
err := CopyDir(r.Fremote, r.Flocal, false)
err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
fs.Config.DryRun = false
require.NoError(t, err)
@@ -54,9 +55,9 @@ func TestCopy(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
err := CopyDir(r.Fremote, r.Flocal, false)
err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -73,7 +74,7 @@ func TestCopyNoTraverse(t *testing.T) {
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := CopyDir(r.Fremote, r.Flocal, false)
err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -91,7 +92,7 @@ func TestSyncNoTraverse(t *testing.T) {
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -109,7 +110,7 @@ func TestCopyWithDepth(t *testing.T) {
fs.Config.MaxDepth = 1
defer func() { fs.Config.MaxDepth = -1 }()
err := CopyDir(r.Fremote, r.Flocal, false)
err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file2)
@@ -140,7 +141,7 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
}
defer unpatch()
err = CopyDir(r.Fremote, r.Flocal, false)
err = CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
unpatch()
@@ -155,11 +156,11 @@ func TestCopyEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(r.Flocal, "sub dir2")
err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")
require.NoError(t, err)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
err = CopyDir(r.Fremote, r.Flocal, true)
err = CopyDir(context.Background(), r.Fremote, r.Flocal, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -181,11 +182,11 @@ func TestMoveEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(r.Flocal, "sub dir2")
err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")
require.NoError(t, err)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
err = MoveDir(r.Fremote, r.Flocal, false, true)
err = MoveDir(context.Background(), r.Fremote, r.Flocal, false, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -207,11 +208,11 @@ func TestSyncEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(r.Flocal, "sub dir2")
err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")
require.NoError(t, err)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
err = Sync(r.Fremote, r.Flocal, true)
err = Sync(context.Background(), r.Fremote, r.Flocal, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -232,7 +233,7 @@ func TestSyncEmptyDirectories(t *testing.T) {
func TestServerSideCopy(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("sub dir/hello world", "hello world", t1)
file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)
fstest.CheckItems(t, r.Fremote, file1)
FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
@@ -240,7 +241,7 @@ func TestServerSideCopy(t *testing.T) {
defer finaliseCopy()
t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)
err = CopyDir(FremoteCopy, r.Fremote, false)
err = CopyDir(context.Background(), FremoteCopy, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, FremoteCopy, file1)
@@ -251,14 +252,14 @@ func TestServerSideCopy(t *testing.T) {
func TestCopyAfterDelete(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("sub dir/hello world", "hello world", t1)
file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)
fstest.CheckItems(t, r.Flocal)
fstest.CheckItems(t, r.Fremote, file1)
err := operations.Mkdir(r.Flocal, "")
err := operations.Mkdir(context.Background(), r.Flocal, "")
require.NoError(t, err)
err = CopyDir(r.Fremote, r.Flocal, false)
err = CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
@@ -269,10 +270,10 @@ func TestCopyAfterDelete(t *testing.T) {
func TestCopyRedownload(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("sub dir/hello world", "hello world", t1)
file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)
fstest.CheckItems(t, r.Fremote, file1)
err := CopyDir(r.Flocal, r.Fremote, false)
err := CopyDir(context.Background(), r.Flocal, r.Fremote, false)
require.NoError(t, err)
// Test with combined precision of local and remote as we copied it there and back
@@ -292,7 +293,7 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@@ -304,7 +305,7 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@@ -326,7 +327,7 @@ func TestSyncSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@@ -338,7 +339,7 @@ func TestSyncSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@@ -360,7 +361,7 @@ func TestSyncIgnoreSize(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@@ -372,7 +373,7 @@ func TestSyncIgnoreSize(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@@ -384,11 +385,11 @@ func TestSyncIgnoreSize(t *testing.T) {
func TestSyncIgnoreTimes(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("existing", "potato", t1)
file1 := r.WriteBoth(context.Background(), "existing", "potato", t1)
fstest.CheckItems(t, r.Fremote, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly 0 files because the
@@ -399,7 +400,7 @@ func TestSyncIgnoreTimes(t *testing.T) {
defer func() { fs.Config.IgnoreTimes = false }()
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file even though the
@@ -419,7 +420,7 @@ func TestSyncIgnoreExisting(t *testing.T) {
defer func() { fs.Config.IgnoreExisting = false }()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
@@ -427,7 +428,7 @@ func TestSyncIgnoreExisting(t *testing.T) {
// Change everything
r.WriteFile("existing", "newpotatoes", t2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// Items should not change
fstest.CheckItems(t, r.Fremote, file1)
@@ -441,9 +442,9 @@ func TestSyncIgnoreErrors(t *testing.T) {
r.Finalise()
}()
file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(r.Fremote, "d"))
file2 := r.WriteObject(context.Background(), "b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(context.Background(), "c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "d"))
fstest.CheckListingWithPrecision(
t,
@@ -475,7 +476,7 @@ func TestSyncIgnoreErrors(t *testing.T) {
accounting.Stats.ResetCounters()
fs.CountError(errors.New("boom"))
assert.NoError(t, Sync(r.Fremote, r.Flocal, false))
assert.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false))
fstest.CheckListingWithPrecision(
t,
@@ -509,7 +510,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("empty space", "", t2)
file2 := r.WriteObject("empty space", "", t1)
file2 := r.WriteObject(context.Background(), "empty space", "", t1)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
@@ -518,7 +519,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
defer func() { fs.Config.DryRun = false }()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -527,7 +528,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
fs.Config.DryRun = false
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -549,13 +550,13 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
}()
file1 := r.WriteFile("empty space", "", t2)
file2 := r.WriteObject("empty space", "", t1)
file2 := r.WriteObject(context.Background(), "empty space", "", t1)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -570,13 +571,13 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {
}
file1 := r.WriteFile("foo", "foo", t2)
file2 := r.WriteObject("foo", "bar", t1)
file2 := r.WriteObject(context.Background(), "foo", "bar", t1)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -589,14 +590,14 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {
func TestSyncAfterAddingAFile(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "empty space", "", t2)
file2 := r.WriteFile("potato", "------------------------------------------------------------", t3)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1, file2)
@@ -605,13 +606,13 @@ func TestSyncAfterAddingAFile(t *testing.T) {
func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("potato", "------------------------------------------------------------", t3)
file1 := r.WriteObject(context.Background(), "potato", "------------------------------------------------------------", t3)
file2 := r.WriteFile("potato", "smaller but same date", t3)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file2)
@@ -625,16 +626,16 @@ func TestSyncAfterChangingContentsOnly(t *testing.T) {
var file1 fstest.Item
if r.Fremote.Precision() == fs.ModTimeNotSupported {
t.Logf("ModTimeNotSupported so forcing file to be a different size")
file1 = r.WriteObject("potato", "different size to make sure it syncs", t3)
file1 = r.WriteObject(context.Background(), "potato", "different size to make sure it syncs", t3)
} else {
file1 = r.WriteObject("potato", "smaller but same date", t3)
file1 = r.WriteObject(context.Background(), "potato", "smaller but same date", t3)
}
file2 := r.WriteFile("potato", "SMALLER BUT SAME DATE", t2)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file2)
@@ -645,12 +646,12 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("empty space", "", t2)
file2 := r.WriteObject(context.Background(), "potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(context.Background(), "empty space", "", t2)
fs.Config.DryRun = true
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
fs.Config.DryRun = false
require.NoError(t, err)
@@ -663,13 +664,13 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("empty space", "", t2)
file2 := r.WriteObject(context.Background(), "potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(context.Background(), "empty space", "", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1, file3)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file3)
fstest.CheckItems(t, r.Fremote, file1, file3)
@@ -680,10 +681,10 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(r.Fremote, "d"))
require.NoError(t, operations.Mkdir(r.Fremote, "d/e"))
file2 := r.WriteObject(context.Background(), "b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(context.Background(), "c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "d"))
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "d/e"))
fstest.CheckListingWithPrecision(
t,
@@ -715,7 +716,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -751,9 +752,9 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject("b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth("c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(r.Fremote, "d"))
file2 := r.WriteObject(context.Background(), "b/potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(context.Background(), "c/non empty space", "AhHa!", t2)
require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "d"))
fstest.CheckListingWithPrecision(
t,
@@ -785,7 +786,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
accounting.Stats.ResetCounters()
fs.CountError(errors.New("boom"))
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
assert.Equal(t, fs.ErrorNotDeleting, err)
fstest.CheckListingWithPrecision(
@@ -856,13 +857,13 @@ func TestCopyDeleteBefore(t *testing.T) {
fs.Config.DeleteMode = fs.DeleteModeDefault
}()
file1 := r.WriteObject("potato", "hopefully not deleted", t1)
file1 := r.WriteObject(context.Background(), "potato", "hopefully not deleted", t1)
file2 := r.WriteFile("potato2", "hopefully copied in", t1)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := CopyDir(r.Fremote, r.Flocal, false)
err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file1, file2)
@@ -873,8 +874,8 @@ func TestCopyDeleteBefore(t *testing.T) {
func TestSyncWithExclude(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fstest.CheckItems(t, r.Fremote, file1, file2)
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
@@ -885,14 +886,14 @@ func TestSyncWithExclude(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file1)
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
accounting.Stats.ResetCounters()
err = Sync(r.Flocal, r.Fremote, false)
err = Sync(context.Background(), r.Flocal, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2, file1, file3)
}
@@ -901,9 +902,9 @@ func TestSyncWithExclude(t *testing.T) {
func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1) // 60 bytes
file2 := r.WriteBoth("empty space", "", t2)
file3 := r.WriteBoth("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1) // 60 bytes
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
file3 := r.WriteBoth(context.Background(), "enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
@@ -915,14 +916,14 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2)
// Check sync the other way round to make sure enormous gets
// deleted even though it is excluded
accounting.Stats.ResetCounters()
err = Sync(r.Flocal, r.Fremote, false)
err = Sync(context.Background(), r.Flocal, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
}
@@ -942,10 +943,10 @@ func TestSyncWithUpdateOlder(t *testing.T) {
fourF := r.WriteFile("four", "four", t2)
fiveF := r.WriteFile("five", "five", t2)
fstest.CheckItems(t, r.Flocal, oneF, twoF, threeF, fourF, fiveF)
oneO := r.WriteObject("one", "ONE", t2)
twoO := r.WriteObject("two", "TWO", t2)
threeO := r.WriteObject("three", "THREE", t2plus)
fourO := r.WriteObject("four", "FOURFOUR", t2minus)
oneO := r.WriteObject(context.Background(), "one", "ONE", t2)
twoO := r.WriteObject(context.Background(), "two", "TWO", t2)
threeO := r.WriteObject(context.Background(), "three", "THREE", t2plus)
fourO := r.WriteObject(context.Background(), "four", "FOURFOUR", t2minus)
fstest.CheckItems(t, r.Fremote, oneO, twoO, threeO, fourO)
fs.Config.UpdateOlder = true
@@ -957,7 +958,7 @@ func TestSyncWithUpdateOlder(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, oneO, twoF, threeO, fourF, fiveF)
}
@@ -981,7 +982,7 @@ func TestSyncWithTrackRenames(t *testing.T) {
f2 := r.WriteFile("yam", "Yam Content", t2)
accounting.Stats.ResetCounters()
require.NoError(t, Sync(r.Fremote, r.Flocal, false))
require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false))
fstest.CheckItems(t, r.Fremote, f1, f2)
fstest.CheckItems(t, r.Flocal, f1, f2)
@@ -990,7 +991,7 @@ func TestSyncWithTrackRenames(t *testing.T) {
f2 = r.RenameFile(f2, "yaml")
accounting.Stats.ResetCounters()
require.NoError(t, Sync(r.Fremote, r.Flocal, false))
require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false))
fstest.CheckItems(t, r.Fremote, f1, f2)
@@ -1015,12 +1016,12 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
require.NoError(t, err)
defer finaliseMove()
file1 := r.WriteBoth("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth("empty space", "", t2)
file3u := r.WriteBoth("potato3", "------------------------------------------------------------ UPDATED", t2)
file1 := r.WriteBoth(context.Background(), "potato2", "------------------------------------------------------------", t1)
file2 := r.WriteBoth(context.Background(), "empty space", "", t2)
file3u := r.WriteBoth(context.Background(), "potato3", "------------------------------------------------------------ UPDATED", t2)
if testDeleteEmptyDirs {
err := operations.Mkdir(r.Fremote, "tomatoDir")
err := operations.Mkdir(context.Background(), r.Fremote, "tomatoDir")
require.NoError(t, err)
}
@@ -1029,13 +1030,13 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
t.Logf("Server side move (if possible) %v -> %v", r.Fremote, FremoteMove)
// Write just one file in the new remote
r.WriteObjectTo(FremoteMove, "empty space", "", t2, false)
file3 := r.WriteObjectTo(FremoteMove, "potato3", "------------------------------------------------------------", t1, false)
r.WriteObjectTo(context.Background(), FremoteMove, "empty space", "", t2, false)
file3 := r.WriteObjectTo(context.Background(), FremoteMove, "potato3", "------------------------------------------------------------", t1, false)
fstest.CheckItems(t, FremoteMove, file2, file3)
// Do server side move
accounting.Stats.ResetCounters()
err = MoveDir(FremoteMove, r.Fremote, testDeleteEmptyDirs, false)
err = MoveDir(context.Background(), FremoteMove, r.Fremote, testDeleteEmptyDirs, false)
require.NoError(t, err)
if withFilter {
@@ -1056,13 +1057,13 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
defer finaliseMove2()
if testDeleteEmptyDirs {
err := operations.Mkdir(FremoteMove, "tomatoDir")
err := operations.Mkdir(context.Background(), FremoteMove, "tomatoDir")
require.NoError(t, err)
}
// Move it back to a new empty remote, dst does not exist this time
accounting.Stats.ResetCounters()
err = MoveDir(FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)
err = MoveDir(context.Background(), FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)
require.NoError(t, err)
if withFilter {
@@ -1084,10 +1085,10 @@ func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
// run move with --delete-empty-src-dirs
err := MoveDir(r.Fremote, r.Flocal, true, false)
err := MoveDir(context.Background(), r.Fremote, r.Flocal, true, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -1105,9 +1106,9 @@ func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
err := MoveDir(r.Fremote, r.Flocal, false, false)
err := MoveDir(context.Background(), r.Fremote, r.Flocal, false, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@@ -1164,11 +1165,11 @@ func TestServerSideMoveOverlap(t *testing.T) {
FremoteMove, err := fs.NewFs(subRemoteName)
require.NoError(t, err)
file1 := r.WriteObject("potato2", "------------------------------------------------------------", t1)
file1 := r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.Fremote, file1)
// Subdir move with no filters should return ErrorCantMoveOverlapping
err = MoveDir(FremoteMove, r.Fremote, false, false)
err = MoveDir(context.Background(), FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
@@ -1176,7 +1177,7 @@ func TestServerSideMoveOverlap(t *testing.T) {
defer func() {
filter.Active.Opt.MinSize = -1
}()
err = MoveDir(FremoteMove, r.Fremote, false, false)
err = MoveDir(context.Background(), FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
}
@@ -1195,10 +1196,10 @@ func TestSyncOverlap(t *testing.T) {
assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())
}
checkErr(Sync(FremoteSync, r.Fremote, false))
checkErr(Sync(r.Fremote, FremoteSync, false))
checkErr(Sync(r.Fremote, r.Fremote, false))
checkErr(Sync(FremoteSync, FremoteSync, false))
checkErr(Sync(context.Background(), FremoteSync, r.Fremote, false))
checkErr(Sync(context.Background(), r.Fremote, FremoteSync, false))
checkErr(Sync(context.Background(), r.Fremote, r.Fremote, false))
checkErr(Sync(context.Background(), FremoteSync, FremoteSync, false))
}
// Test with BackupDir set
@@ -1209,7 +1210,7 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
if !operations.CanServerSideMove(r.Fremote) {
t.Skip("Skipping test as remote does not support server side move")
}
r.Mkdir(r.Fremote)
r.Mkdir(context.Background(), r.Fremote)
fs.Config.BackupDir = r.FremoteName + "/backup"
fs.Config.Suffix = suffix
@@ -1222,9 +1223,9 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
// Make the setup so we have one, two, three in the dest
// and one (different), two (same) in the source
file1 := r.WriteObject("dst/one", "one", t1)
file2 := r.WriteObject("dst/two", "two", t1)
file3 := r.WriteObject("dst/three.txt", "three", t1)
file1 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "dst/two", "two", t1)
file3 := r.WriteObject(context.Background(), "dst/three.txt", "three", t1)
file2a := r.WriteFile("two", "two", t1)
file1a := r.WriteFile("one", "oneA", t2)
@@ -1235,7 +1236,7 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
require.NoError(t, err)
accounting.Stats.ResetCounters()
err = Sync(fdst, r.Flocal, false)
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
@@ -1253,14 +1254,14 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
// Now check what happens if we do it again
// Restore a different three and update one in the source
file3a := r.WriteObject("dst/three.txt", "threeA", t2)
file3a := r.WriteObject(context.Background(), "dst/three.txt", "threeA", t2)
file1b := r.WriteFile("one", "oneBB", t3)
fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a, file3a)
// This should delete three and overwrite one again, checking
// the files got overwritten correctly in backup-dir
accounting.Stats.ResetCounters()
err = Sync(fdst, r.Flocal, false)
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
@@ -1298,11 +1299,11 @@ func TestSyncUTFNorm(t *testing.T) {
file1 := r.WriteFile(Encoding1, "This is a test", t1)
fstest.CheckItems(t, r.Flocal, file1)
file2 := r.WriteObject(Encoding2, "This is a old test", t2)
file2 := r.WriteObject(context.Background(), Encoding2, "This is a old test", t2)
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file, but kept the
@@ -1328,7 +1329,7 @@ func TestSyncImmutable(t *testing.T) {
// Should succeed
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
@@ -1340,7 +1341,7 @@ func TestSyncImmutable(t *testing.T) {
// Should fail with ErrorImmutableModified and not modify local or remote files
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal, false)
err = Sync(context.Background(), r.Fremote, r.Flocal, false)
assert.EqualError(t, err, fs.ErrorImmutableModified.Error())
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file1)
@@ -1362,12 +1363,12 @@ func TestSyncIgnoreCase(t *testing.T) {
// Create files with different filename casing
file1 := r.WriteFile("existing", "potato", t1)
fstest.CheckItems(t, r.Flocal, file1)
file2 := r.WriteObject("EXISTING", "potato", t1)
file2 := r.WriteObject(context.Background(), "EXISTING", "potato", t1)
fstest.CheckItems(t, r.Fremote, file2)
// Should not copy files that are differently-cased but otherwise identical
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
@@ -1403,6 +1404,6 @@ func TestAbort(t *testing.T) {
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal, false)
err := Sync(context.Background(), r.Fremote, r.Flocal, false)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
}

69
fs/walk/walk.go
View File

@@ -3,6 +3,7 @@ package walk
import (
"bytes"
"context"
"fmt"
"path"
"sort"
@@ -58,15 +59,15 @@ type Func func(path string, entries fs.DirEntries, err error) error
// constructed with just those files in and then walked with WalkR
//
// NB (f, path) to be replaced by fs.Dir at some point
func Walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
return walkR(f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(f.NewObject))
return walkR(ctx, f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(ctx, f.NewObject))
}
// FIXME should this just be maxLevel < 0 - why the maxLevel > 1
if (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && f.Features().ListR != nil {
return walkListR(f, path, includeAll, maxLevel, fn)
return walkListR(ctx, f, path, includeAll, maxLevel, fn)
}
return walkListDirSorted(f, path, includeAll, maxLevel, fn)
return walkListDirSorted(ctx, f, path, includeAll, maxLevel, fn)
}
// ListType is uses to choose which combination of files or directories is requires
@@ -137,7 +138,7 @@ func (l ListType) Filter(in *fs.DirEntries) {
// efficient, otherwise by Walk.
//
// NB (f, path) to be replaced by fs.Dir at some point
func ListR(f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error {
func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error {
// FIXME disable this with --no-fast-list ??? `--disable ListR` will do it...
doListR := f.Features().ListR
@@ -147,15 +148,15 @@ func ListR(f fs.Fs, path string, includeAll bool, maxLevel int, listType ListTyp
maxLevel >= 0 || // ...using bounded recursion
len(filter.Active.Opt.ExcludeFile) > 0 || // ...using --exclude-file
filter.Active.BoundedRecursion() { // ...filters imply bounded recursion
return listRwalk(f, path, includeAll, maxLevel, listType, fn)
return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
}
return listR(f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
}
// listRwalk walks the file tree for ListR using Walk
func listRwalk(f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error {
func listRwalk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listType ListType, fn fs.ListRCallback) error {
var listErr error
walkErr := Walk(f, path, includeAll, maxLevel, func(path string, entries fs.DirEntries, err error) error {
walkErr := Walk(ctx, f, path, includeAll, maxLevel, func(path string, entries fs.DirEntries, err error) error {
// Carry on listing but return the error at the end
if err != nil {
listErr = err
@@ -264,8 +265,8 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) {
}
// listR walks the file tree using ListR
func listR(f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error {
includeDirectory := filter.Active.IncludeDirectory(f)
func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListRCallback, doListR fs.ListRFn, synthesizeDirs bool) error {
includeDirectory := filter.Active.IncludeDirectory(ctx, f)
if !includeAll {
includeAll = filter.Active.InActive()
}
@@ -274,7 +275,7 @@ func listR(f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListR
dm = newDirMap(path)
}
var mu sync.Mutex
err := doListR(path, func(entries fs.DirEntries) (err error) {
err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
if synthesizeDirs {
err = dm.addEntries(entries)
if err != nil {
@@ -288,7 +289,7 @@ func listR(f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListR
var include bool
switch x := entry.(type) {
case fs.Object:
include = filter.Active.IncludeObject(x)
include = filter.Active.IncludeObject(ctx, x)
case fs.Directory:
include, err = includeDirectory(x.Remote())
if err != nil {
@@ -324,25 +325,25 @@ func listR(f fs.Fs, path string, includeAll bool, listType ListType, fn fs.ListR
// walkListDirSorted lists the directory.
//
// It implements Walk using non recursive directory listing.
func walkListDirSorted(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
return walk(f, path, includeAll, maxLevel, fn, list.DirSorted)
func walkListDirSorted(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
return walk(ctx, f, path, includeAll, maxLevel, fn, list.DirSorted)
}
// walkListR lists the directory.
//
// It implements Walk using recursive directory listing if
// available, or returns ErrorCantListR if not.
func walkListR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
func walkListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
listR := f.Features().ListR
if listR == nil {
return ErrorCantListR
}
return walkR(f, path, includeAll, maxLevel, fn, listR)
return walkR(ctx, f, path, includeAll, maxLevel, fn, listR)
}
type listDirFunc func(fs fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)
type listDirFunc func(ctx context.Context, fs fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)
func walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error {
func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error {
var (
wg sync.WaitGroup // sync closing of go routines
traversing sync.WaitGroup // running directory traversals
@@ -378,7 +379,7 @@ func walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir
if !ok {
return
}
entries, err := listDir(f, includeAll, job.remote)
entries, err := listDir(ctx, f, includeAll, job.remote)
var jobs []listJob
if err == nil && job.depth != 0 {
entries.ForDir(func(dir fs.Directory) {
@@ -608,14 +609,14 @@ func (dt DirTree) String() string {
return out.String()
}
func walkRDirTree(f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (DirTree, error) {
func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll bool, maxLevel int, listR fs.ListRFn) (DirTree, error) {
dirs := make(DirTree)
// Entries can come in arbitrary order. We use toPrune to keep
// all directories to exclude later.
toPrune := make(map[string]bool)
includeDirectory := filter.Active.IncludeDirectory(f)
includeDirectory := filter.Active.IncludeDirectory(ctx, f)
var mu sync.Mutex
err := listR(startPath, func(entries fs.DirEntries) error {
err := listR(ctx, startPath, func(entries fs.DirEntries) error {
mu.Lock()
defer mu.Unlock()
for _, entry := range entries {
@@ -623,7 +624,7 @@ func walkRDirTree(f fs.Fs, startPath string, includeAll bool, maxLevel int, list
switch x := entry.(type) {
case fs.Object:
// Make sure we don't delete excluded files if not required
if includeAll || filter.Active.IncludeObject(x) {
if includeAll || filter.Active.IncludeObject(ctx, x) {
if maxLevel < 0 || slashes <= maxLevel-1 {
dirs.add(x)
} else {
@@ -685,7 +686,7 @@ func walkRDirTree(f fs.Fs, startPath string, includeAll bool, maxLevel int, list
}
// Create a DirTree using List
func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
func walkNDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
dirs := make(DirTree)
fn := func(dirPath string, entries fs.DirEntries, err error) error {
if err == nil {
@@ -693,7 +694,7 @@ func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir l
}
return err
}
err := walk(f, path, includeAll, maxLevel, fn, listDir)
err := walk(ctx, f, path, includeAll, maxLevel, fn, listDir)
if err != nil {
return nil, err
}
@@ -715,18 +716,18 @@ func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir l
// constructed with just those files in.
//
// NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(f fs.Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
return walkRDirTree(f, path, includeAll, maxLevel, filter.Active.MakeListR(f.NewObject))
return walkRDirTree(ctx, f, path, includeAll, maxLevel, filter.Active.MakeListR(ctx, f.NewObject))
}
if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil {
return walkRDirTree(f, path, includeAll, maxLevel, ListR)
return walkRDirTree(ctx, f, path, includeAll, maxLevel, ListR)
}
return walkNDirTree(f, path, includeAll, maxLevel, list.DirSorted)
return walkNDirTree(ctx, f, path, includeAll, maxLevel, list.DirSorted)
}
func walkR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR fs.ListRFn) error {
dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
func walkR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR fs.ListRFn) error {
dirs, err := walkRDirTree(ctx, f, path, includeAll, maxLevel, listR)
if err != nil {
return err
}
@@ -760,8 +761,8 @@ func walkR(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listR f
}
// GetAll runs ListR getting all the results
func GetAll(f fs.Fs, path string, includeAll bool, maxLevel int) (objs []fs.Object, dirs []fs.Directory, err error) {
err = ListR(f, path, includeAll, maxLevel, ListAll, func(entries fs.DirEntries) error {
func GetAll(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (objs []fs.Object, dirs []fs.Directory, err error) {
err = ListR(ctx, f, path, includeAll, maxLevel, ListAll, func(entries fs.DirEntries) error {
for _, entry := range entries {
switch x := entry.(type) {
case fs.Object:

43
fs/walk/walk_test.go
View File

@@ -1,6 +1,7 @@
package walk
import (
"context"
"fmt"
"io"
"strings"
@@ -68,7 +69,7 @@ func (ls *listDirs) SetLevel(maxLevel int) *listDirs {
}
// ListDir returns the expected listing for the directory
func (ls *listDirs) ListDir(f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
func (ls *listDirs) ListDir(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
ls.mu.Lock()
defer ls.mu.Unlock()
assert.Equal(ls.t, ls.fs, f)
@@ -89,7 +90,7 @@ func (ls *listDirs) ListDir(f fs.Fs, includeAll bool, dir string) (entries fs.Di
}
// ListR returns the expected listing for the directory using ListR
func (ls *listDirs) ListR(dir string, callback fs.ListRCallback) (err error) {
func (ls *listDirs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
ls.mu.Lock()
defer ls.mu.Unlock()
@@ -150,14 +151,14 @@ func (ls *listDirs) WalkFn(dir string, entries fs.DirEntries, err error) error {
// Walk does the walk and tests the expectations
func (ls *listDirs) Walk() {
err := walk(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
err := walk(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListDir)
assert.Equal(ls.t, ls.finalError, err)
ls.IsFinished()
}
// WalkR does the walkR and tests the expectations
func (ls *listDirs) WalkR() {
err := walkR(nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
err := walkR(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.WalkFn, ls.ListR)
assert.Equal(ls.t, ls.finalError, err)
if ls.finalError == nil {
ls.IsFinished()
@@ -266,7 +267,7 @@ func TestWalkRLevelsNoRecursive10(t *testing.T) { testWalkLevels(t, 10).WalkR()
func TestWalkNDirTree(t *testing.T) {
ls := testWalkLevels(t, -1)
entries, err := walkNDirTree(nil, "", ls.includeAll, ls.maxLevel, ls.ListDir)
entries, err := walkNDirTree(context.Background(), nil, "", ls.includeAll, ls.maxLevel, ls.ListDir)
require.NoError(t, err)
assert.Equal(t, `/
A
@@ -414,7 +415,7 @@ func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() }
// a very simple listRcallback function
func makeListRCallback(entries fs.DirEntries, err error) fs.ListRFn {
return func(dir string, callback fs.ListRCallback) error {
return func(ctx context.Context, dir string, callback fs.ListRCallback) error {
if err == nil {
err = callback(entries)
}
@@ -559,7 +560,7 @@ a/
b/
`, nil, "", 2},
} {
r, err := walkRDirTree(nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
r, err := walkRDirTree(context.Background(), nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
}
@@ -630,7 +631,7 @@ b/c/d/
`, nil, "", -1, "ign", true},
} {
filter.Active.Opt.ExcludeFile = test.excludeFile
r, err := walkRDirTree(nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err))
r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err))
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
}
@@ -701,7 +702,7 @@ func TestListR(t *testing.T) {
}
return nil
}
doListR := func(dir string, callback fs.ListRCallback) error {
doListR := func(ctx context.Context, dir string, callback fs.ListRCallback) error {
var os fs.DirEntries
for _, o := range objects {
if dir == "" || strings.HasPrefix(o.Remote(), dir+"/") {
@@ -725,43 +726,43 @@ func TestListR(t *testing.T) {
// Base case
clearCallback()
err = listR(f, "", true, ListAll, callback, doListR, false)
err = listR(context.Background(), f, "", true, ListAll, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir", "dir/a", "dir/b", "dir/c"}, got)
// Base case - with Objects
clearCallback()
err = listR(f, "", true, ListObjects, callback, doListR, false)
err = listR(context.Background(), f, "", true, ListObjects, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/c"}, got)
// Base case - with Dirs
clearCallback()
err = listR(f, "", true, ListDirs, callback, doListR, false)
err = listR(context.Background(), f, "", true, ListDirs, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"dir"}, got)
// With filter
clearCallback()
err = listR(f, "", false, ListAll, callback, doListR, false)
err = listR(context.Background(), f, "", false, ListAll, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"b", "dir", "dir/b"}, got)
// With filter - with Objects
clearCallback()
err = listR(f, "", false, ListObjects, callback, doListR, false)
err = listR(context.Background(), f, "", false, ListObjects, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"b", "dir/b"}, got)
// With filter - with Dir
clearCallback()
err = listR(f, "", false, ListDirs, callback, doListR, false)
err = listR(context.Background(), f, "", false, ListDirs, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"dir"}, got)
// With filter and subdir
clearCallback()
err = listR(f, "dir", false, ListAll, callback, doListR, false)
err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, false)
require.NoError(t, err)
require.Equal(t, []string{"dir/b"}, got)
@@ -777,31 +778,31 @@ func TestListR(t *testing.T) {
// Base case
clearCallback()
err = listR(f, "", true, ListAll, callback, doListR, true)
err = listR(context.Background(), f, "", true, ListAll, callback, doListR, true)
require.NoError(t, err)
require.Equal(t, []string{"a", "b", "dir/a", "dir/b", "dir/subdir/c", "dir/subdir", "dir"}, got)
// With filter
clearCallback()
err = listR(f, "", false, ListAll, callback, doListR, true)
err = listR(context.Background(), f, "", false, ListAll, callback, doListR, true)
require.NoError(t, err)
require.Equal(t, []string{"b", "dir/b", "dir/subdir", "dir"}, got)
// With filter and subdir
clearCallback()
err = listR(f, "dir", false, ListAll, callback, doListR, true)
err = listR(context.Background(), f, "dir", false, ListAll, callback, doListR, true)
require.NoError(t, err)
require.Equal(t, []string{"dir/b", "dir/subdir"}, got)
// With filter and subdir - with Objects
clearCallback()
err = listR(f, "dir", false, ListObjects, callback, doListR, true)
err = listR(context.Background(), f, "dir", false, ListObjects, callback, doListR, true)
require.NoError(t, err)
require.Equal(t, []string{"dir/b"}, got)
// With filter and subdir - with Dirs
clearCallback()
err = listR(f, "dir", false, ListDirs, callback, doListR, true)
err = listR(context.Background(), f, "dir", false, ListDirs, callback, doListR, true)
require.NoError(t, err)
require.Equal(t, []string{"dir/subdir"}, got)
}