mirror of
https://github.com/rclone/rclone.git
synced 2025-12-15 15:53:41 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
c6dfd5f2d3 | ||
|
|
99695d57ab | ||
|
|
ca3752f824 | ||
|
|
d0ca58bbb1 | ||
|
|
580fa3a5a7 |
65
README.md
65
README.md
@@ -7,7 +7,7 @@ Rclone
|
||||
|
||||
[](http://rclone.org/)
|
||||
|
||||
Sync files and directories to and from
|
||||
Rclone is a command line program to sync files and directories to and from
|
||||
|
||||
* Google Drive
|
||||
* Amazon S3
|
||||
@@ -43,15 +43,13 @@ Or alternatively if you have Go installed use
|
||||
|
||||
and this will build the binary in `$GOPATH/bin`.
|
||||
|
||||
You can then modify the source and submit patches.
|
||||
|
||||
Configure
|
||||
---------
|
||||
|
||||
First you'll need to configure rclone. As the object storage systems
|
||||
have quite complicated authentication these are kept in a config file
|
||||
`.rclone.conf` in your home directory by default. (You can use the
|
||||
-config option to choose a different config file.)
|
||||
`--config` option to choose a different config file.)
|
||||
|
||||
The easiest way to make the config is to run rclone with the config
|
||||
option, Eg
|
||||
@@ -63,7 +61,7 @@ Usage
|
||||
|
||||
Rclone syncs a directory tree from local to remote.
|
||||
|
||||
Its basic syntax is like this
|
||||
Its basic syntax is
|
||||
|
||||
Syntax: [options] subcommand <parameters> <parameters...>
|
||||
|
||||
@@ -84,7 +82,7 @@ Sync the source to the destination. Doesn't transfer
|
||||
unchanged files, testing first by modification time then by
|
||||
MD5SUM. Deletes any files that exist in source that don't
|
||||
exist in destination. Since this can cause data loss, test
|
||||
first with the -dry-run flag.
|
||||
first with the `--dry-run` flag.
|
||||
|
||||
rclone ls [remote:path]
|
||||
|
||||
@@ -92,7 +90,7 @@ List all the objects in the the path.
|
||||
|
||||
rclone lsd [remote:path]
|
||||
|
||||
List all directoryes/objects/buckets in the the path.
|
||||
List all directories/objects/buckets in the the path.
|
||||
|
||||
rclone mkdir remote:path
|
||||
|
||||
@@ -114,17 +112,23 @@ compares sizes and MD5SUMs and prints a report of files which
|
||||
don't match. It doesn't alter the source or destination.
|
||||
|
||||
General options:
|
||||
* `-config` Location of the config file
|
||||
* `-transfers=4`: Number of file transfers to run in parallel.
|
||||
* `-checkers=8`: Number of MD5SUM checkers to run in parallel.
|
||||
* `-dry-run=false`: Do a trial run with no permanent changes
|
||||
* `-modify-window=1ns`: Max time difference to be considered the same - this is automatically set usually
|
||||
* `-quiet=false`: Print as little stuff as possible
|
||||
* `-stats=1m0s`: Interval to print stats
|
||||
* `-verbose=false`: Print lots more stuff
|
||||
|
||||
```
|
||||
--checkers=8: Number of checkers to run in parallel.
|
||||
--config="~/.rclone.conf": Config file.
|
||||
-n, --dry-run=false: Do a trial run with no permanent changes
|
||||
--modify-window=1ns: Max time diff to be considered the same
|
||||
-q, --quiet=false: Print as little stuff as possible
|
||||
--stats=1m0s: Interval to print stats
|
||||
--transfers=4: Number of file transfers to run in parallel.
|
||||
-v, --verbose=false: Print lots more stuff
|
||||
```
|
||||
|
||||
Developer options:
|
||||
* `-cpuprofile=""`: Write cpu profile to file
|
||||
|
||||
```
|
||||
--cpuprofile="": Write cpu profile to file
|
||||
```
|
||||
|
||||
Local Filesystem
|
||||
----------------
|
||||
@@ -133,13 +137,14 @@ Paths are specified as normal filesystem paths, so
|
||||
|
||||
rclone sync /home/source /tmp/destination
|
||||
|
||||
Will sync source to destination
|
||||
Will sync `/home/source` to `/tmp/destination`
|
||||
|
||||
Swift / Rackspace cloudfiles / Memset Memstore
|
||||
----------------------------------------------
|
||||
|
||||
Paths are specified as remote:container (or remote: for the `lsd`
|
||||
command.)
|
||||
command.) You may put subdirectories in too, eg
|
||||
`remote:container/path/to/dir`.
|
||||
|
||||
So to copy a local directory to a swift container called backup:
|
||||
|
||||
@@ -155,7 +160,8 @@ os.Stat) for an object.
|
||||
Amazon S3
|
||||
---------
|
||||
|
||||
Paths are specified as remote:bucket
|
||||
Paths are specified as remote:bucket. You may put subdirectories in
|
||||
too, eg `remote:bucket/path/to/dir`.
|
||||
|
||||
So to copy a local directory to a s3 container called backup
|
||||
|
||||
@@ -170,7 +176,7 @@ Google drive
|
||||
Paths are specified as drive:path Drive paths may be as deep as required.
|
||||
|
||||
The initial setup for drive involves getting a token from Google drive
|
||||
which you need to do in your browser. The `rclone config` walks you
|
||||
which you need to do in your browser. `rclone config` walks you
|
||||
through it.
|
||||
|
||||
To copy a local directory to a drive directory called backup
|
||||
@@ -179,6 +185,19 @@ To copy a local directory to a drive directory called backup
|
||||
|
||||
Google drive stores modification times accurate to 1 ms.
|
||||
|
||||
Single file copies
|
||||
------------------
|
||||
|
||||
Rclone can copy single files
|
||||
|
||||
rclone src:path/to/file dst:path/dir
|
||||
|
||||
Or
|
||||
|
||||
rclone src:path/to/file dst:path/to/file
|
||||
|
||||
Note that you can't rename the file if you are copying from one file to another.
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
@@ -188,7 +207,6 @@ COPYING file included in this package).
|
||||
Bugs
|
||||
----
|
||||
|
||||
* Doesn't sync individual files yet, only directories.
|
||||
* Drive: Sometimes get: Failed to copy: Upload failed: googleapi: Error 403: Rate Limit Exceeded
|
||||
* quota is 100.0 requests/second/user
|
||||
* Empty directories left behind with Local and Drive
|
||||
@@ -197,6 +215,9 @@ Bugs
|
||||
Changelog
|
||||
---------
|
||||
|
||||
* v0.97 - 2014-05-05
|
||||
* Implement copying of single files
|
||||
* s3 & swift: support paths inside containers/buckets
|
||||
* v0.96 - 2014-04-24
|
||||
* drive: Fix multiple files of same name being created
|
||||
* drive: Use o.Update and fs.Put to optimise transfers
|
||||
@@ -228,7 +249,7 @@ The project website is at:
|
||||
|
||||
* https://github.com/ncw/rclone
|
||||
|
||||
There you can file bug reports, ask for help or contribute patches.
|
||||
There you can file bug reports, ask for help or send pull requests.
|
||||
|
||||
Authors
|
||||
-------
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
title: "Rclone"
|
||||
description: "rclone syncs files to and from Google Drive, S3, Swift and Cloudfiles."
|
||||
type: page
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
groups: ["about"]
|
||||
---
|
||||
|
||||
@@ -11,7 +11,7 @@ Rclone
|
||||
|
||||
[](http://rclone.org/)
|
||||
|
||||
Sync files and directories to and from
|
||||
Rclone is a command line program to sync files and directories to and from
|
||||
|
||||
* Google Drive
|
||||
* Amazon S3
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
---
|
||||
title: "Contact"
|
||||
description: "Contact the rclone project"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Contact the rclone project
|
||||
|
||||
* [Github project page for source and reporting bugs](http://github.com/ncw/rclone)
|
||||
* [Github project page for source, reporting bugs and pull requests](http://github.com/ncw/rclone)
|
||||
* <a href="https://plus.google.com/110609214444437761115" rel="publisher">Google+ page for general comments</a></li>
|
||||
|
||||
Or email [Nick Craig-Wood](mailto:nick@craig-wood.com)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Documentation"
|
||||
description: "Rclone Documentation"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Install
|
||||
@@ -9,7 +9,7 @@ Install
|
||||
|
||||
Rclone is a Go program and comes as a single binary file.
|
||||
|
||||
[Download the relevant binary.](/downloads/)
|
||||
[Download](/downloads/) the relevant binary.
|
||||
|
||||
Or alternatively if you have Go installed use
|
||||
|
||||
@@ -17,15 +17,13 @@ Or alternatively if you have Go installed use
|
||||
|
||||
and this will build the binary in `$GOPATH/bin`.
|
||||
|
||||
You can then modify the source and submit patches.
|
||||
|
||||
Configure
|
||||
---------
|
||||
|
||||
First you'll need to configure rclone. As the object storage systems
|
||||
have quite complicated authentication these are kept in a config file
|
||||
`.rclone.conf` in your home directory by default. (You can use the
|
||||
`-config` option to choose a different config file.)
|
||||
`--config` option to choose a different config file.)
|
||||
|
||||
The easiest way to make the config is to run rclone with the config
|
||||
option:
|
||||
|
||||
@@ -2,34 +2,34 @@
|
||||
title: "Rclone downloads"
|
||||
description: "Download rclone binaries for your OS."
|
||||
type: page
|
||||
date: "2014-04-25"
|
||||
date: "2014-05-05"
|
||||
---
|
||||
|
||||
v0.96
|
||||
v0.97
|
||||
=====
|
||||
|
||||
* Windows
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-windows-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-windows-amd64.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-windows-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-windows-amd64.zip)
|
||||
* OSX
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-osx-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-osx-amd64.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-osx-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-osx-amd64.zip)
|
||||
* Linux
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-linux-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-linux-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.96-linux-arm.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-linux-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-linux-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.97-linux-arm.zip)
|
||||
* FreeBSD
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-freebsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-freebsd-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.96-freebsd-arm.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-freebsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-freebsd-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.97-freebsd-arm.zip)
|
||||
* NetBSD
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-netbsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-netbsd-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.96-netbsd-arm.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-netbsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-netbsd-amd64.zip)
|
||||
* [ARM - 32 Bit](http://downloads.rclone.org/rclone-v0.97-netbsd-arm.zip)
|
||||
* OpenBSD
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-openbsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.96-openbsd-amd64.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-openbsd-386.zip)
|
||||
* [AMD64 - 64 Bit](http://downloads.rclone.org/rclone-v0.97-openbsd-amd64.zip)
|
||||
* Plan 9
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.96-plan9-386.zip)
|
||||
* [386 - 32 Bit](http://downloads.rclone.org/rclone-v0.97-plan9-386.zip)
|
||||
|
||||
Older downloads can be found [here](http://downloads.rclone.org/)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Google drive"
|
||||
description: "Rclone docs for Google drive"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Paths are specified as `drive:path`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Local Filesystem"
|
||||
description: "Rclone docs for the local filesystem"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Local Filesystem
|
||||
@@ -19,7 +19,7 @@ but it is probably easier not to.
|
||||
Modified time
|
||||
-------------
|
||||
|
||||
We read and write the modified time using an accuracy determined by
|
||||
Rclone reads and writes the modified time using an accuracy determined by
|
||||
the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second
|
||||
on OS X.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Amazon S3"
|
||||
description: "Rclone docs for Amazon S3"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Paths are specified as `remote:bucket` or `remote:`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
title: "Swift"
|
||||
description: "Swift"
|
||||
date: "2014-03-19"
|
||||
date: "2014-04-26"
|
||||
---
|
||||
|
||||
Swift refers to [Openstack Object Storage](http://www.openstack.org/software/openstack-storage/).
|
||||
|
||||
@@ -135,14 +135,15 @@ func (name tokenCache) PutToken(token *oauth.Token) error {
|
||||
|
||||
// FsDrive represents a remote drive server
|
||||
type FsDrive struct {
|
||||
svc *drive.Service // the connection to the drive server
|
||||
root string // the path we are working on
|
||||
client *http.Client // authorized client
|
||||
about *drive.About // information about the drive, including the root
|
||||
rootId string // Id of the root directory
|
||||
foundRoot sync.Once // Whether we need to find the root directory or not
|
||||
dirCache dirCache // Map of directory path to directory id
|
||||
findDirLock sync.Mutex // Protect findDir from concurrent use
|
||||
svc *drive.Service // the connection to the drive server
|
||||
root string // the path we are working on
|
||||
client *http.Client // authorized client
|
||||
about *drive.About // information about the drive, including the root
|
||||
rootId string // Id of the root directory
|
||||
foundRoot bool // Whether we have found the root or not
|
||||
findRootLock sync.Mutex // Protect findRoot from concurrent use
|
||||
dirCache dirCache // Map of directory path to directory id
|
||||
findDirLock sync.Mutex // Protect findDir from concurrent use
|
||||
}
|
||||
|
||||
// FsObjectDrive describes a drive object
|
||||
@@ -305,7 +306,10 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &FsDrive{root: root, dirCache: newDirCache()}
|
||||
f := &FsDrive{
|
||||
root: root,
|
||||
dirCache: newDirCache(),
|
||||
}
|
||||
|
||||
// Try to pull the token from the cache; if this fails, we need to get one.
|
||||
token, err := t.Config.TokenCache.Token()
|
||||
@@ -331,14 +335,33 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||
f.rootId = f.about.RootFolderId
|
||||
// Put the root directory in
|
||||
f.dirCache.Put("", f.rootId)
|
||||
// Find the current root
|
||||
err = f.findRoot(false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := splitPath(root)
|
||||
newF := *f
|
||||
newF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = newF.findRoot(false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
obj, err := newF.newFsObjectWithInfo(remote, nil)
|
||||
if err != nil {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
// return a Fs Limited to this object
|
||||
return fs.NewLimited(&newF, obj), nil
|
||||
}
|
||||
// fmt.Printf("Root id %s", f.rootId)
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
|
||||
func (f *FsDrive) newFsObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
|
||||
fs := &FsObjectDrive{
|
||||
drive: f,
|
||||
remote: remote,
|
||||
@@ -349,9 +372,18 @@ func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object
|
||||
err := fs.readMetaData() // reads info and meta, returning an error
|
||||
if err != nil {
|
||||
// logged already fs.Debug("Failed to read info: %s", err)
|
||||
return nil
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func (f *FsDrive) NewFsObjectWithInfo(remote string, info *drive.File) fs.Object {
|
||||
fs, _ := f.newFsObjectWithInfo(remote, info)
|
||||
// Errors have already been logged
|
||||
return fs
|
||||
}
|
||||
|
||||
@@ -585,14 +617,21 @@ func (f *FsDrive) _findDir(path string, create bool) (pathId string, err error)
|
||||
//
|
||||
// If create is set it will make the directory if not found
|
||||
func (f *FsDrive) findRoot(create bool) error {
|
||||
var err error
|
||||
f.foundRoot.Do(func() {
|
||||
f.rootId, err = f.findDir(f.root, create)
|
||||
f.dirCache.Flush()
|
||||
// Put the root directory in
|
||||
f.dirCache.Put("", f.rootId)
|
||||
})
|
||||
return err
|
||||
f.findRootLock.Lock()
|
||||
defer f.findRootLock.Unlock()
|
||||
if f.foundRoot {
|
||||
return nil
|
||||
}
|
||||
rootId, err := f.findDir(f.root, create)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.rootId = rootId
|
||||
f.dirCache.Flush()
|
||||
// Put the root directory in
|
||||
f.dirCache.Put("", f.rootId)
|
||||
f.foundRoot = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
|
||||
12
fs/fs.go
12
fs/fs.go
@@ -18,9 +18,15 @@ var (
|
||||
|
||||
// Filesystem info
|
||||
type FsInfo struct {
|
||||
Name string // name of this fs
|
||||
NewFs func(string, string) (Fs, error) // create a new file system
|
||||
Config func(string) // function to call to help with config
|
||||
// Name of this fs
|
||||
Name string
|
||||
// Create a new file system. If root refers to an existing
|
||||
// object, then it should return a Fs which only returns that
|
||||
// object.
|
||||
NewFs func(name string, root string) (Fs, error)
|
||||
// Function to call to help with config
|
||||
Config func(string)
|
||||
// Options for the Fs configuration
|
||||
Options []Option
|
||||
}
|
||||
|
||||
|
||||
88
fs/limited.go
Normal file
88
fs/limited.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This defines a Limited Fs which can only return the Objects passed in from the Fs passed in
|
||||
type Limited struct {
|
||||
objects []Object
|
||||
fs Fs
|
||||
}
|
||||
|
||||
// NewLimited maks a limited Fs limited to the objects passed in
|
||||
func NewLimited(fs Fs, objects ...Object) Fs {
|
||||
f := &Limited{
|
||||
objects: objects,
|
||||
fs: fs,
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Limited) String() string {
|
||||
return fmt.Sprintf("%s limited to %d objects", f.fs.String(), len(f.objects))
|
||||
}
|
||||
|
||||
// List the Fs into a channel
|
||||
func (f *Limited) List() ObjectsChan {
|
||||
out := make(ObjectsChan, Config.Checkers)
|
||||
go func() {
|
||||
for _, obj := range f.objects {
|
||||
out <- obj
|
||||
}
|
||||
close(out)
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
// List the Fs directories/buckets/containers into a channel
|
||||
func (f *Limited) ListDir() DirChan {
|
||||
out := make(DirChan, Config.Checkers)
|
||||
close(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Find the Object at remote. Returns nil if can't be found
|
||||
func (f *Limited) NewFsObject(remote string) Object {
|
||||
for _, obj := range f.objects {
|
||||
if obj.Remote() == remote {
|
||||
return obj
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Limited) Put(in io.Reader, remote string, modTime time.Time, size int64) (Object, error) {
|
||||
obj := f.NewFsObject(remote)
|
||||
if obj == nil {
|
||||
return nil, fmt.Errorf("Can't create %q in limited fs", remote)
|
||||
}
|
||||
return obj, obj.Update(in, modTime, size)
|
||||
}
|
||||
|
||||
// Make the directory (container, bucket)
|
||||
func (f *Limited) Mkdir() error {
|
||||
// All directories are already made - just ignore
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove the directory (container, bucket) if empty
|
||||
func (f *Limited) Rmdir() error {
|
||||
return fmt.Errorf("Can't rmdir in limited fs")
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Limited) Precision() time.Duration {
|
||||
return f.fs.Precision()
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var _ Fs = &Limited{}
|
||||
@@ -45,6 +45,16 @@ type FsObjectLocal struct {
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
root = path.Clean(root)
|
||||
f := &FsLocal{root: root}
|
||||
// Check to see if this points to a file
|
||||
fi, err := os.Lstat(f.root)
|
||||
if err == nil && fi.Mode().IsRegular() {
|
||||
// It is a file, so use the parent as the root
|
||||
remote := path.Base(root)
|
||||
f.root = path.Dir(root)
|
||||
obj := f.NewFsObject(remote)
|
||||
// return a Fs Limited to this object
|
||||
return fs.NewLimited(f, obj), nil
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
Todo
|
||||
* FIXME: ls without an argument for buckets/containers?
|
||||
* FIXME: More -dry-run checks for object transfer
|
||||
* Might be quicker to check md5sums first? for swift <-> swift certainly, and maybe for small files
|
||||
* swift: Ignoring the pseudo directories
|
||||
@@ -12,7 +11,6 @@ Todo
|
||||
* make Account do progress meter
|
||||
* Make logging controllable with flags (mostly done)
|
||||
* -timeout: Make all timeouts be settable with command line parameters
|
||||
* Check the locking in swift module!
|
||||
* Windows paths? Do we need to translate / and \?
|
||||
* Make a fs.Errorf and count errors and log them at a different level
|
||||
* Add max object size to fs metadata - 5GB for swift, infinite for local, ? for s3
|
||||
@@ -22,7 +20,6 @@ Ideas
|
||||
* could do encryption - put IV into metadata?
|
||||
* optimise remote copy container to another container using remote
|
||||
copy if local is same as remote - use an optional Copier interface
|
||||
* Allow subpaths container:/sub/path
|
||||
* support
|
||||
* sftp
|
||||
* scp
|
||||
@@ -35,6 +32,8 @@ Need to make directory objects otherwise can't upload an empty directory
|
||||
* Or could upload empty directories only?
|
||||
* Can't purge a local filesystem because it leaves the directories behind
|
||||
|
||||
Copying a single file? Or maybe with a glob pattern? Could do with LimitedFs
|
||||
|
||||
s3
|
||||
* Can maybe set last modified?
|
||||
* https://forums.aws.amazon.com/message.jspa?messageID=214062
|
||||
@@ -43,6 +42,7 @@ s3
|
||||
|
||||
Bugs
|
||||
* Non verbose - not sure number transferred got counted up? CHECK
|
||||
* When doing copy it recurses the whole of the destination FS which isn't necessary
|
||||
|
||||
Making a release
|
||||
* go build ./...
|
||||
|
||||
@@ -106,7 +106,7 @@ var Commands = []Command{
|
||||
Name: "lsd",
|
||||
ArgsHelp: "[remote://path]",
|
||||
Help: `
|
||||
List all directoryes/objects/buckets in the the path.`,
|
||||
List all directories/containers/buckets in the the path.`,
|
||||
Run: func(fdst, fsrc fs.Fs) {
|
||||
err := fs.ListDir(fdst)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
package main
|
||||
|
||||
const Version = "v0.96"
|
||||
const Version = "v0.97"
|
||||
|
||||
148
s3/s3.go
148
s3/s3.go
@@ -7,7 +7,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path"
|
||||
@@ -111,6 +110,7 @@ type FsS3 struct {
|
||||
b *s3.Bucket // the connection to the bucket
|
||||
bucket string // the bucket we are working on
|
||||
perm s3.ACL // permissions for new buckets / objects
|
||||
root string // root of the bucket - ignore all objects above this
|
||||
}
|
||||
|
||||
// FsObjectS3 describes a s3 object
|
||||
@@ -131,7 +131,10 @@ type FsObjectS3 struct {
|
||||
|
||||
// String converts this FsS3 to a string
|
||||
func (f *FsS3) String() string {
|
||||
return fmt.Sprintf("S3 bucket %s", f.bucket)
|
||||
if f.root == "" {
|
||||
return fmt.Sprintf("S3 bucket %s", f.bucket)
|
||||
}
|
||||
return fmt.Sprintf("S3 bucket %s path %s", f.bucket, f.root)
|
||||
}
|
||||
|
||||
// Pattern to match a s3 path
|
||||
@@ -185,14 +188,11 @@ func s3Connection(name string) (*s3.S3, error) {
|
||||
}
|
||||
|
||||
// NewFsS3 contstructs an FsS3 from the path, bucket:path
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
bucket, directory, err := s3ParsePath(path)
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
bucket, directory, err := s3ParsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q: %q", path, directory)
|
||||
}
|
||||
c, err := s3Connection(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -202,6 +202,24 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||
bucket: bucket,
|
||||
b: c.Bucket(bucket),
|
||||
perm: s3.Private, // FIXME need user to specify
|
||||
root: directory,
|
||||
}
|
||||
if f.root != "" {
|
||||
f.root += "/"
|
||||
// Check to see if the object exists
|
||||
_, err = f.b.Head(directory, nil)
|
||||
if err == nil {
|
||||
remote := path.Base(directory)
|
||||
f.root = path.Dir(directory)
|
||||
if f.root == "." {
|
||||
f.root = ""
|
||||
} else {
|
||||
f.root += "/"
|
||||
}
|
||||
obj := f.NewFsObject(remote)
|
||||
// return a Fs Limited to this object
|
||||
return fs.NewLimited(f, obj), nil
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
@@ -241,48 +259,100 @@ func (f *FsS3) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsS3) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
// FIXME need to implement ALL loop
|
||||
objects, err := f.b.List("", "", "", 10000)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Printf("Couldn't read bucket %q: %s", f.bucket, err)
|
||||
// list the objects into the function supplied
|
||||
//
|
||||
// If directories is set it only sends directories
|
||||
func (f *FsS3) list(directories bool, fn func(string, *s3.Key)) {
|
||||
delimiter := ""
|
||||
if directories {
|
||||
delimiter = "/"
|
||||
}
|
||||
// FIXME need to implement ALL loop
|
||||
objects, err := f.b.List(f.root, delimiter, "", 10000)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Couldn't read bucket %q: %s", f.bucket, err)
|
||||
} else {
|
||||
rootLength := len(f.root)
|
||||
if directories {
|
||||
for _, remote := range objects.CommonPrefixes {
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
fs.Log(f, "Odd name received %q", remote)
|
||||
continue
|
||||
}
|
||||
remote := remote[rootLength:]
|
||||
fn(remote, &s3.Key{Key: remote})
|
||||
}
|
||||
} else {
|
||||
for i := range objects.Contents {
|
||||
object := &objects.Contents[i]
|
||||
if fs := f.NewFsObjectWithInfo(object.Key, object); fs != nil {
|
||||
out <- fs
|
||||
if !strings.HasPrefix(object.Key, f.root) {
|
||||
fs.Log(f, "Odd name received %q", object.Key)
|
||||
continue
|
||||
}
|
||||
remote := object.Key[rootLength:]
|
||||
fn(remote, object)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsS3) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
if f.bucket == "" {
|
||||
// Return no objects at top level list
|
||||
close(out)
|
||||
}()
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Can't list objects at root - choose a bucket using lsd")
|
||||
} else {
|
||||
go func() {
|
||||
defer close(out)
|
||||
f.list(false, func(remote string, object *s3.Key) {
|
||||
if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
|
||||
out <- fs
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Lists the buckets
|
||||
func (f *FsS3) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
buckets, err := f.c.ListBuckets()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Printf("Couldn't list buckets: %s", err)
|
||||
} else {
|
||||
for _, bucket := range buckets {
|
||||
out <- &fs.Dir{
|
||||
Name: bucket.Name,
|
||||
When: bucket.CreationDate,
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
if f.bucket == "" {
|
||||
// List the buckets
|
||||
go func() {
|
||||
defer close(out)
|
||||
buckets, err := f.c.ListBuckets()
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Couldn't list buckets: %s", err)
|
||||
} else {
|
||||
for _, bucket := range buckets {
|
||||
out <- &fs.Dir{
|
||||
Name: bucket.Name,
|
||||
When: bucket.CreationDate,
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}()
|
||||
} else {
|
||||
// List the directories in the path in the bucket
|
||||
go func() {
|
||||
defer close(out)
|
||||
f.list(true, func(remote string, object *s3.Key) {
|
||||
out <- &fs.Dir{
|
||||
Name: remote,
|
||||
Bytes: object.Size,
|
||||
Count: 0,
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -354,7 +424,7 @@ func (o *FsObjectS3) readMetaData() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
headers, err := o.s3.b.Head(o.remote, nil)
|
||||
headers, err := o.s3.b.Head(o.s3.root+o.remote, nil)
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to read info: %s", err)
|
||||
return err
|
||||
@@ -407,7 +477,7 @@ func (o *FsObjectS3) SetModTime(modTime time.Time) {
|
||||
return
|
||||
}
|
||||
o.meta[metaMtime] = swift.TimeToFloatString(modTime)
|
||||
_, err = o.s3.b.Update(o.remote, o.s3.perm, o.meta)
|
||||
_, err = o.s3.b.Update(o.s3.root+o.remote, o.s3.perm, o.meta)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(o, "Failed to update remote mtime: %s", err)
|
||||
@@ -421,7 +491,7 @@ func (o *FsObjectS3) Storable() bool {
|
||||
|
||||
// Open an object for read
|
||||
func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
|
||||
in, err = o.s3.b.GetReader(o.remote)
|
||||
in, err = o.s3.b.GetReader(o.s3.root + o.remote)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -438,13 +508,13 @@ func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
_, err := o.s3.b.PutReaderHeaders(o.remote, in, size, contentType, o.s3.perm, headers)
|
||||
_, err := o.s3.b.PutReaderHeaders(o.s3.root+o.remote, in, size, contentType, o.s3.perm, headers)
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *FsObjectS3) Remove() error {
|
||||
return o.s3.b.Del(o.remote)
|
||||
return o.s3.b.Del(o.s3.root + o.remote)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
|
||||
164
swift/swift.go
164
swift/swift.go
@@ -1,13 +1,11 @@
|
||||
// Swift interface
|
||||
package swift
|
||||
|
||||
// FIXME need to prevent anything but ListDir working for swift://
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -73,7 +71,10 @@ type FsObjectSwift struct {
|
||||
|
||||
// String converts this FsSwift to a string
|
||||
func (f *FsSwift) String() string {
|
||||
return fmt.Sprintf("Swift container %s", f.container)
|
||||
if f.root == "" {
|
||||
return fmt.Sprintf("Swift container %s", f.container)
|
||||
}
|
||||
return fmt.Sprintf("Swift container %s path %s", f.container, f.root)
|
||||
}
|
||||
|
||||
// Pattern to match a swift path
|
||||
@@ -118,19 +119,37 @@ func swiftConnection(name string) (*swift.Connection, error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an FsSwift from the path, container:path
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
container, directory, err := parsePath(path)
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
container, directory, err := parsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if directory != "" {
|
||||
return nil, fmt.Errorf("Directories not supported yet in %q", path)
|
||||
}
|
||||
c, err := swiftConnection(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &FsSwift{c: *c, container: container, root: directory}
|
||||
f := &FsSwift{
|
||||
c: *c,
|
||||
container: container,
|
||||
root: directory,
|
||||
}
|
||||
if f.root != "" {
|
||||
f.root += "/"
|
||||
// Check to see if the object exists
|
||||
_, _, err = f.c.Object(container, directory)
|
||||
if err == nil {
|
||||
remote := path.Base(directory)
|
||||
f.root = path.Dir(directory)
|
||||
if f.root == "." {
|
||||
f.root = ""
|
||||
} else {
|
||||
f.root += "/"
|
||||
}
|
||||
obj := f.NewFsObject(remote)
|
||||
// return a Fs Limited to this object
|
||||
return fs.NewLimited(f, obj), nil
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
@@ -162,51 +181,100 @@ func (f *FsSwift) NewFsObject(remote string) fs.Object {
|
||||
return f.NewFsObjectWithInfo(remote, nil)
|
||||
}
|
||||
|
||||
// list the objects into the function supplied
|
||||
//
|
||||
// If directories is set it only sends directories
|
||||
func (f *FsSwift) list(directories bool, fn func(string, *swift.Object)) {
|
||||
// Options for ObjectsWalk
|
||||
opts := swift.ObjectsOpts{
|
||||
Prefix: f.root,
|
||||
Limit: 256,
|
||||
}
|
||||
if directories {
|
||||
opts.Delimiter = '/'
|
||||
}
|
||||
rootLength := len(f.root)
|
||||
err := f.c.ObjectsWalk(f.container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) {
|
||||
objects, err := f.c.Objects(f.container, opts)
|
||||
if err == nil {
|
||||
for i := range objects {
|
||||
object := &objects[i]
|
||||
// FIXME if there are no directories, swift gives back the files for some reason!
|
||||
if directories && !strings.HasSuffix(object.Name, "/") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(object.Name, f.root) {
|
||||
fs.Log(f, "Odd name received %q", object.Name)
|
||||
continue
|
||||
}
|
||||
remote := object.Name[rootLength:]
|
||||
fn(remote, object)
|
||||
}
|
||||
}
|
||||
return objects, err
|
||||
})
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Couldn't read container %q: %s", f.container, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
func (f *FsSwift) List() fs.ObjectsChan {
|
||||
out := make(fs.ObjectsChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
// FIXME use a smaller limit?
|
||||
err := f.c.ObjectsWalk(f.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
|
||||
objects, err := f.c.Objects(f.container, opts)
|
||||
if err == nil {
|
||||
for i := range objects {
|
||||
object := &objects[i]
|
||||
if fs := f.NewFsObjectWithInfo(object.Name, object); fs != nil {
|
||||
out <- fs
|
||||
}
|
||||
}
|
||||
}
|
||||
return objects, err
|
||||
})
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Printf("Couldn't read container %q: %s", f.container, err)
|
||||
}
|
||||
if f.container == "" {
|
||||
// Return no objects at top level list
|
||||
close(out)
|
||||
}()
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Can't list objects at root - choose a container using lsd")
|
||||
} else {
|
||||
// List the objects
|
||||
go func() {
|
||||
defer close(out)
|
||||
f.list(false, func(remote string, object *swift.Object) {
|
||||
if fs := f.NewFsObjectWithInfo(remote, object); fs != nil {
|
||||
out <- fs
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Lists the containers
|
||||
func (f *FsSwift) ListDir() fs.DirChan {
|
||||
out := make(fs.DirChan, fs.Config.Checkers)
|
||||
go func() {
|
||||
defer close(out)
|
||||
containers, err := f.c.ContainersAll(nil)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
log.Printf("Couldn't list containers: %s", err)
|
||||
} else {
|
||||
for _, container := range containers {
|
||||
out <- &fs.Dir{
|
||||
Name: container.Name,
|
||||
Bytes: container.Bytes,
|
||||
Count: container.Count,
|
||||
if f.container == "" {
|
||||
// List the containers
|
||||
go func() {
|
||||
defer close(out)
|
||||
containers, err := f.c.ContainersAll(nil)
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(f, "Couldn't list containers: %v", err)
|
||||
} else {
|
||||
for _, container := range containers {
|
||||
out <- &fs.Dir{
|
||||
Name: container.Name,
|
||||
Bytes: container.Bytes,
|
||||
Count: container.Count,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}()
|
||||
} else {
|
||||
// List the directories in the path in the container
|
||||
go func() {
|
||||
defer close(out)
|
||||
f.list(true, func(remote string, object *swift.Object) {
|
||||
out <- &fs.Dir{
|
||||
Name: remote,
|
||||
Bytes: object.Bytes,
|
||||
Count: 0,
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -275,7 +343,7 @@ func (o *FsObjectSwift) readMetaData() (err error) {
|
||||
if o.meta != nil {
|
||||
return nil
|
||||
}
|
||||
info, h, err := o.swift.c.Object(o.swift.container, o.remote)
|
||||
info, h, err := o.swift.c.Object(o.swift.container, o.swift.root+o.remote)
|
||||
if err != nil {
|
||||
fs.Debug(o, "Failed to read info: %s", err)
|
||||
return err
|
||||
@@ -314,7 +382,7 @@ func (o *FsObjectSwift) SetModTime(modTime time.Time) {
|
||||
return
|
||||
}
|
||||
o.meta.SetModTime(modTime)
|
||||
err = o.swift.c.ObjectUpdate(o.swift.container, o.remote, o.meta.ObjectHeaders())
|
||||
err = o.swift.c.ObjectUpdate(o.swift.container, o.swift.root+o.remote, o.meta.ObjectHeaders())
|
||||
if err != nil {
|
||||
fs.Stats.Error()
|
||||
fs.Log(o, "Failed to update remote mtime: %s", err)
|
||||
@@ -328,7 +396,7 @@ func (o *FsObjectSwift) Storable() bool {
|
||||
|
||||
// Open an object for read
|
||||
func (o *FsObjectSwift) Open() (in io.ReadCloser, err error) {
|
||||
in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.remote, true, nil)
|
||||
in, _, err = o.swift.c.ObjectOpen(o.swift.container, o.swift.root+o.remote, true, nil)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -339,13 +407,13 @@ func (o *FsObjectSwift) Update(in io.Reader, modTime time.Time, size int64) erro
|
||||
// Set the mtime
|
||||
m := swift.Metadata{}
|
||||
m.SetModTime(modTime)
|
||||
_, err := o.swift.c.ObjectPut(o.swift.container, o.remote, in, true, "", "", m.ObjectHeaders())
|
||||
_, err := o.swift.c.ObjectPut(o.swift.container, o.swift.root+o.remote, in, true, "", "", m.ObjectHeaders())
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *FsObjectSwift) Remove() error {
|
||||
return o.swift.c.ObjectDelete(o.swift.container, o.remote)
|
||||
return o.swift.c.ObjectDelete(o.swift.container, o.swift.root+o.remote)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
package main
|
||||
|
||||
const Version = "v0.96"
|
||||
const Version = "v0.97"
|
||||
|
||||
Reference in New Issue
Block a user