1
0
mirror of https://github.com/rclone/rclone.git synced 2025-12-16 00:04:40 +00:00

Compare commits

..

4 Commits

Author SHA1 Message Date
Nick Craig-Wood
cbe049b520 cmount: support --mount-tpslimit flag
See: https://forum.rclone.org/t/mac-os-big-sur-mount-error/21726
2021-02-07 13:01:07 +00:00
Nick Craig-Wood
a662d17105 mountlib: add --mount-tpslimit flag 2021-02-07 12:57:21 +00:00
Nick Craig-Wood
e32f08f37b drive: refer to Shared Drives instead of Team Drives 2021-02-07 12:30:21 +00:00
Nick Craig-Wood
fea4b753b2 Add Alex JOST to contributors 2021-02-07 12:30:21 +00:00
8 changed files with 99 additions and 79 deletions

View File

@@ -207,7 +207,7 @@ func init() {
} }
err = configTeamDrive(ctx, opt, m, name) err = configTeamDrive(ctx, opt, m, name)
if err != nil { if err != nil {
log.Fatalf("Failed to configure team drive: %v", err) log.Fatalf("Failed to configure Shared Drive: %v", err)
} }
}, },
Options: append(driveOAuthOptions(), []fs.Option{{ Options: append(driveOAuthOptions(), []fs.Option{{
@@ -247,7 +247,7 @@ a non root folder as its starting point.
Advanced: true, Advanced: true,
}, { }, {
Name: "team_drive", Name: "team_drive",
Help: "ID of the Team Drive", Help: "ID of the Shared Drive (Team Drive)",
Hide: fs.OptionHideConfigurator, Hide: fs.OptionHideConfigurator,
Advanced: true, Advanced: true,
}, { }, {
@@ -666,7 +666,7 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
fs.Errorf(f, "Received download limit error: %v", err) fs.Errorf(f, "Received download limit error: %v", err)
return false, fserrors.FatalError(err) return false, fserrors.FatalError(err)
} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" { } else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
fs.Errorf(f, "Received team drive file limit error: %v", err) fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
return false, fserrors.FatalError(err) return false, fserrors.FatalError(err)
} }
} }
@@ -955,24 +955,24 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
return nil return nil
} }
if opt.TeamDriveID == "" { if opt.TeamDriveID == "" {
fmt.Printf("Configure this as a team drive?\n") fmt.Printf("Configure this as a Shared Drive (Team Drive)?\n")
} else { } else {
fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID) fmt.Printf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID)
} }
if !config.Confirm(false) { if !config.Confirm(false) {
return nil return nil
} }
f, err := newFs(ctx, name, "", m) f, err := newFs(ctx, name, "", m)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to make Fs to list teamdrives") return errors.Wrap(err, "failed to make Fs to list Shared Drives")
} }
fmt.Printf("Fetching team drive list...\n") fmt.Printf("Fetching Shared Drive list...\n")
teamDrives, err := f.listTeamDrives(ctx) teamDrives, err := f.listTeamDrives(ctx)
if err != nil { if err != nil {
return err return err
} }
if len(teamDrives) == 0 { if len(teamDrives) == 0 {
fmt.Printf("No team drives found in your account") fmt.Printf("No Shared Drives found in your account")
return nil return nil
} }
var driveIDs, driveNames []string var driveIDs, driveNames []string
@@ -980,7 +980,7 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
driveIDs = append(driveIDs, teamDrive.Id) driveIDs = append(driveIDs, teamDrive.Id)
driveNames = append(driveNames, teamDrive.Name) driveNames = append(driveNames, teamDrive.Name)
} }
driveID := config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true) driveID := config.Choose("Enter a Shared Drive ID", driveIDs, driveNames, true)
m.Set("team_drive", driveID) m.Set("team_drive", driveID)
m.Set("root_folder_id", "") m.Set("root_folder_id", "")
opt.TeamDriveID = driveID opt.TeamDriveID = driveID
@@ -2475,9 +2475,9 @@ func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
return f.shouldRetry(err) return f.shouldRetry(err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to get Team/Shared Drive info") return errors.Wrap(err, "failed to get Shared Drive info")
} }
fs.Debugf(f, "read info from team drive %q", td.Name) fs.Debugf(f, "read info from Shared Drive %q", td.Name)
return err return err
} }
@@ -2963,7 +2963,7 @@ func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err
return defaultFs.shouldRetry(err) return defaultFs.shouldRetry(err)
}) })
if err != nil { if err != nil {
return drives, errors.Wrap(err, "listing team drives failed") return drives, errors.Wrap(err, "listing Team Drives failed")
} }
drives = append(drives, teamDrives.TeamDrives...) drives = append(drives, teamDrives.TeamDrives...)
if teamDrives.NextPageToken == "" { if teamDrives.NextPageToken == "" {
@@ -3131,8 +3131,8 @@ authenticated with "drive2:" can't read files from "drive:".
}, },
}, { }, {
Name: "drives", Name: "drives",
Short: "List the shared drives available to this account", Short: "List the Shared Drives available to this account",
Long: `This command lists the shared drives (teamdrives) available to this Long: `This command lists the Shared Drives (Team Drives) available to this
account. account.
Usage: Usage:

View File

@@ -5,6 +5,7 @@
package cmount package cmount
import ( import (
"context"
"io" "io"
"os" "os"
"path" "path"
@@ -18,9 +19,11 @@ import (
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/log" "github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs"
"golang.org/x/time/rate"
) )
const fhUnset = ^uint64(0) const fhUnset = ^uint64(0)
const tpsBurst = 100
// FS represents the top level filing system // FS represents the top level filing system
type FS struct { type FS struct {
@@ -29,19 +32,31 @@ type FS struct {
ready chan (struct{}) ready chan (struct{})
mu sync.Mutex // to protect the below mu sync.Mutex // to protect the below
handles []vfs.Handle handles []vfs.Handle
destroyed int32 // read/write with sync/atomic destroyed int32 // read/write with sync/atomic
tps *rate.Limiter // for limiting number of transactions per second
} }
// NewFS makes a new FS // NewFS makes a new FS
func NewFS(VFS *vfs.VFS) *FS { func NewFS(VFS *vfs.VFS, opt *mountlib.Options) *FS {
fsys := &FS{ fsys := &FS{
VFS: VFS, VFS: VFS,
f: VFS.Fs(), f: VFS.Fs(),
ready: make(chan (struct{})), ready: make(chan (struct{})),
} }
if opt.TPSLimit > 0 {
fsys.tps = rate.NewLimiter(rate.Limit(opt.TPSLimit), tpsBurst)
fs.Infof(nil, "Starting mount transaction limiter: max %g transactions/s with burst %d", opt.TPSLimit, tpsBurst)
}
return fsys return fsys
} }
// Limit the number of transactions per second
func (fsys *FS) rateLimit() {
if fsys.tps != nil {
_ = fsys.tps.Wait(context.Background())
}
}
// Open a handle returning an integer file handle // Open a handle returning an integer file handle
func (fsys *FS) openHandle(handle vfs.Handle) (fh uint64) { func (fsys *FS) openHandle(handle vfs.Handle) (fh uint64) {
fsys.mu.Lock() fsys.mu.Lock()
@@ -195,6 +210,7 @@ func (fsys *FS) Destroy() {
// Getattr reads the attributes for path // Getattr reads the attributes for path
func (fsys *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) { func (fsys *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
defer log.Trace(path, "fh=0x%X", fh)("errc=%v", &errc) defer log.Trace(path, "fh=0x%X", fh)("errc=%v", &errc)
fsys.rateLimit()
node, _, errc := fsys.getNode(path, fh) node, _, errc := fsys.getNode(path, fh)
if errc == 0 { if errc == 0 {
errc = fsys.stat(node, stat) errc = fsys.stat(node, stat)
@@ -205,6 +221,7 @@ func (fsys *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
// Opendir opens path as a directory // Opendir opens path as a directory
func (fsys *FS) Opendir(path string) (errc int, fh uint64) { func (fsys *FS) Opendir(path string) (errc int, fh uint64) {
defer log.Trace(path, "")("errc=%d, fh=0x%X", &errc, &fh) defer log.Trace(path, "")("errc=%d, fh=0x%X", &errc, &fh)
fsys.rateLimit()
handle, err := fsys.VFS.OpenFile(path, os.O_RDONLY, 0777) handle, err := fsys.VFS.OpenFile(path, os.O_RDONLY, 0777)
if err != nil { if err != nil {
return translateError(err), fhUnset return translateError(err), fhUnset
@@ -219,6 +236,7 @@ func (fsys *FS) Readdir(dirPath string,
fh uint64) (errc int) { fh uint64) (errc int) {
itemsRead := -1 itemsRead := -1
defer log.Trace(dirPath, "ofst=%d, fh=0x%X", ofst, fh)("items=%d, errc=%d", &itemsRead, &errc) defer log.Trace(dirPath, "ofst=%d, fh=0x%X", ofst, fh)("items=%d, errc=%d", &itemsRead, &errc)
fsys.rateLimit()
dir, errc := fsys.lookupDir(dirPath) dir, errc := fsys.lookupDir(dirPath)
if errc != 0 { if errc != 0 {
@@ -270,12 +288,14 @@ func (fsys *FS) Readdir(dirPath string,
// Releasedir finished reading the directory // Releasedir finished reading the directory
func (fsys *FS) Releasedir(path string, fh uint64) (errc int) { func (fsys *FS) Releasedir(path string, fh uint64) (errc int) {
defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc) defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc)
fsys.rateLimit()
return fsys.closeHandle(fh) return fsys.closeHandle(fh)
} }
// Statfs reads overall stats on the filesystem // Statfs reads overall stats on the filesystem
func (fsys *FS) Statfs(path string, stat *fuse.Statfs_t) (errc int) { func (fsys *FS) Statfs(path string, stat *fuse.Statfs_t) (errc int) {
defer log.Trace(path, "")("stat=%+v, errc=%d", stat, &errc) defer log.Trace(path, "")("stat=%+v, errc=%d", stat, &errc)
fsys.rateLimit()
const blockSize = 4096 const blockSize = 4096
total, _, free := fsys.VFS.Statfs() total, _, free := fsys.VFS.Statfs()
stat.Blocks = uint64(total) / blockSize // Total data blocks in file system. stat.Blocks = uint64(total) / blockSize // Total data blocks in file system.
@@ -295,6 +315,7 @@ func (fsys *FS) Statfs(path string, stat *fuse.Statfs_t) (errc int) {
// OpenEx opens a file // OpenEx opens a file
func (fsys *FS) OpenEx(path string, fi *fuse.FileInfo_t) (errc int) { func (fsys *FS) OpenEx(path string, fi *fuse.FileInfo_t) (errc int) {
defer log.Trace(path, "flags=0x%X", fi.Flags)("errc=%d, fh=0x%X", &errc, &fi.Fh) defer log.Trace(path, "flags=0x%X", fi.Flags)("errc=%d, fh=0x%X", &errc, &fi.Fh)
fsys.rateLimit()
fi.Fh = fhUnset fi.Fh = fhUnset
// translate the fuse flags to os flags // translate the fuse flags to os flags
@@ -325,6 +346,7 @@ func (fsys *FS) Open(path string, flags int) (errc int, fh uint64) {
// CreateEx creates and opens a file. // CreateEx creates and opens a file.
func (fsys *FS) CreateEx(filePath string, mode uint32, fi *fuse.FileInfo_t) (errc int) { func (fsys *FS) CreateEx(filePath string, mode uint32, fi *fuse.FileInfo_t) (errc int) {
defer log.Trace(filePath, "flags=0x%X, mode=0%o", fi.Flags, mode)("errc=%d, fh=0x%X", &errc, &fi.Fh) defer log.Trace(filePath, "flags=0x%X, mode=0%o", fi.Flags, mode)("errc=%d, fh=0x%X", &errc, &fi.Fh)
fsys.rateLimit()
fi.Fh = fhUnset fi.Fh = fhUnset
leaf, parentDir, errc := fsys.lookupParentDir(filePath) leaf, parentDir, errc := fsys.lookupParentDir(filePath)
if errc != 0 { if errc != 0 {
@@ -356,6 +378,7 @@ func (fsys *FS) Create(filePath string, flags int, mode uint32) (errc int, fh ui
// Truncate truncates a file to size // Truncate truncates a file to size
func (fsys *FS) Truncate(path string, size int64, fh uint64) (errc int) { func (fsys *FS) Truncate(path string, size int64, fh uint64) (errc int) {
defer log.Trace(path, "size=%d, fh=0x%X", size, fh)("errc=%d", &errc) defer log.Trace(path, "size=%d, fh=0x%X", size, fh)("errc=%d", &errc)
fsys.rateLimit()
node, handle, errc := fsys.getNode(path, fh) node, handle, errc := fsys.getNode(path, fh)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -375,6 +398,7 @@ func (fsys *FS) Truncate(path string, size int64, fh uint64) (errc int) {
// Read data from file handle // Read data from file handle
func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) { func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
defer log.Trace(path, "ofst=%d, fh=0x%X", ofst, fh)("n=%d", &n) defer log.Trace(path, "ofst=%d, fh=0x%X", ofst, fh)("n=%d", &n)
fsys.rateLimit()
handle, errc := fsys.getHandle(fh) handle, errc := fsys.getHandle(fh)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -390,6 +414,7 @@ func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
// Write data to file handle // Write data to file handle
func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) { func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
defer log.Trace(path, "ofst=%d, fh=0x%X", ofst, fh)("n=%d", &n) defer log.Trace(path, "ofst=%d, fh=0x%X", ofst, fh)("n=%d", &n)
fsys.rateLimit()
handle, errc := fsys.getHandle(fh) handle, errc := fsys.getHandle(fh)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -404,6 +429,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
// Flush flushes an open file descriptor or path // Flush flushes an open file descriptor or path
func (fsys *FS) Flush(path string, fh uint64) (errc int) { func (fsys *FS) Flush(path string, fh uint64) (errc int) {
defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc) defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc)
fsys.rateLimit()
handle, errc := fsys.getHandle(fh) handle, errc := fsys.getHandle(fh)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -414,6 +440,7 @@ func (fsys *FS) Flush(path string, fh uint64) (errc int) {
// Release closes the file if still open // Release closes the file if still open
func (fsys *FS) Release(path string, fh uint64) (errc int) { func (fsys *FS) Release(path string, fh uint64) (errc int) {
defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc) defer log.Trace(path, "fh=0x%X", fh)("errc=%d", &errc)
fsys.rateLimit()
handle, errc := fsys.getHandle(fh) handle, errc := fsys.getHandle(fh)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -425,6 +452,7 @@ func (fsys *FS) Release(path string, fh uint64) (errc int) {
// Unlink removes a file. // Unlink removes a file.
func (fsys *FS) Unlink(filePath string) (errc int) { func (fsys *FS) Unlink(filePath string) (errc int) {
defer log.Trace(filePath, "")("errc=%d", &errc) defer log.Trace(filePath, "")("errc=%d", &errc)
fsys.rateLimit()
leaf, parentDir, errc := fsys.lookupParentDir(filePath) leaf, parentDir, errc := fsys.lookupParentDir(filePath)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -435,6 +463,7 @@ func (fsys *FS) Unlink(filePath string) (errc int) {
// Mkdir creates a directory. // Mkdir creates a directory.
func (fsys *FS) Mkdir(dirPath string, mode uint32) (errc int) { func (fsys *FS) Mkdir(dirPath string, mode uint32) (errc int) {
defer log.Trace(dirPath, "mode=0%o", mode)("errc=%d", &errc) defer log.Trace(dirPath, "mode=0%o", mode)("errc=%d", &errc)
fsys.rateLimit()
leaf, parentDir, errc := fsys.lookupParentDir(dirPath) leaf, parentDir, errc := fsys.lookupParentDir(dirPath)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -446,6 +475,7 @@ func (fsys *FS) Mkdir(dirPath string, mode uint32) (errc int) {
// Rmdir removes a directory // Rmdir removes a directory
func (fsys *FS) Rmdir(dirPath string) (errc int) { func (fsys *FS) Rmdir(dirPath string) (errc int) {
defer log.Trace(dirPath, "")("errc=%d", &errc) defer log.Trace(dirPath, "")("errc=%d", &errc)
fsys.rateLimit()
leaf, parentDir, errc := fsys.lookupParentDir(dirPath) leaf, parentDir, errc := fsys.lookupParentDir(dirPath)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -456,6 +486,7 @@ func (fsys *FS) Rmdir(dirPath string) (errc int) {
// Rename renames a file. // Rename renames a file.
func (fsys *FS) Rename(oldPath string, newPath string) (errc int) { func (fsys *FS) Rename(oldPath string, newPath string) (errc int) {
defer log.Trace(oldPath, "newPath=%q", newPath)("errc=%d", &errc) defer log.Trace(oldPath, "newPath=%q", newPath)("errc=%d", &errc)
fsys.rateLimit()
return translateError(fsys.VFS.Rename(oldPath, newPath)) return translateError(fsys.VFS.Rename(oldPath, newPath))
} }
@@ -467,6 +498,7 @@ var invalidDateCutoff = time.Date(1601, 1, 2, 0, 0, 0, 0, time.UTC)
// Utimens changes the access and modification times of a file. // Utimens changes the access and modification times of a file.
func (fsys *FS) Utimens(path string, tmsp []fuse.Timespec) (errc int) { func (fsys *FS) Utimens(path string, tmsp []fuse.Timespec) (errc int) {
defer log.Trace(path, "tmsp=%+v", tmsp)("errc=%d", &errc) defer log.Trace(path, "tmsp=%+v", tmsp)("errc=%d", &errc)
fsys.rateLimit()
node, errc := fsys.lookupNode(path) node, errc := fsys.lookupNode(path)
if errc != 0 { if errc != 0 {
return errc return errc
@@ -487,12 +519,14 @@ func (fsys *FS) Utimens(path string, tmsp []fuse.Timespec) (errc int) {
// Mknod creates a file node. // Mknod creates a file node.
func (fsys *FS) Mknod(path string, mode uint32, dev uint64) (errc int) { func (fsys *FS) Mknod(path string, mode uint32, dev uint64) (errc int) {
defer log.Trace(path, "mode=0x%X, dev=0x%X", mode, dev)("errc=%d", &errc) defer log.Trace(path, "mode=0x%X, dev=0x%X", mode, dev)("errc=%d", &errc)
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }
// Fsync synchronizes file contents. // Fsync synchronizes file contents.
func (fsys *FS) Fsync(path string, datasync bool, fh uint64) (errc int) { func (fsys *FS) Fsync(path string, datasync bool, fh uint64) (errc int) {
defer log.Trace(path, "datasync=%v, fh=0x%X", datasync, fh)("errc=%d", &errc) defer log.Trace(path, "datasync=%v, fh=0x%X", datasync, fh)("errc=%d", &errc)
fsys.rateLimit()
// This is a no-op for rclone // This is a no-op for rclone
return 0 return 0
} }
@@ -500,24 +534,28 @@ func (fsys *FS) Fsync(path string, datasync bool, fh uint64) (errc int) {
// Link creates a hard link to a file. // Link creates a hard link to a file.
func (fsys *FS) Link(oldpath string, newpath string) (errc int) { func (fsys *FS) Link(oldpath string, newpath string) (errc int) {
defer log.Trace(oldpath, "newpath=%q", newpath)("errc=%d", &errc) defer log.Trace(oldpath, "newpath=%q", newpath)("errc=%d", &errc)
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }
// Symlink creates a symbolic link. // Symlink creates a symbolic link.
func (fsys *FS) Symlink(target string, newpath string) (errc int) { func (fsys *FS) Symlink(target string, newpath string) (errc int) {
defer log.Trace(target, "newpath=%q", newpath)("errc=%d", &errc) defer log.Trace(target, "newpath=%q", newpath)("errc=%d", &errc)
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }
// Readlink reads the target of a symbolic link. // Readlink reads the target of a symbolic link.
func (fsys *FS) Readlink(path string) (errc int, linkPath string) { func (fsys *FS) Readlink(path string) (errc int, linkPath string) {
defer log.Trace(path, "")("linkPath=%q, errc=%d", &linkPath, &errc) defer log.Trace(path, "")("linkPath=%q, errc=%d", &linkPath, &errc)
fsys.rateLimit()
return -fuse.ENOSYS, "" return -fuse.ENOSYS, ""
} }
// Chmod changes the permission bits of a file. // Chmod changes the permission bits of a file.
func (fsys *FS) Chmod(path string, mode uint32) (errc int) { func (fsys *FS) Chmod(path string, mode uint32) (errc int) {
defer log.Trace(path, "mode=0%o", mode)("errc=%d", &errc) defer log.Trace(path, "mode=0%o", mode)("errc=%d", &errc)
fsys.rateLimit()
// This is a no-op for rclone // This is a no-op for rclone
return 0 return 0
} }
@@ -525,6 +563,7 @@ func (fsys *FS) Chmod(path string, mode uint32) (errc int) {
// Chown changes the owner and group of a file. // Chown changes the owner and group of a file.
func (fsys *FS) Chown(path string, uid uint32, gid uint32) (errc int) { func (fsys *FS) Chown(path string, uid uint32, gid uint32) (errc int) {
defer log.Trace(path, "uid=%d, gid=%d", uid, gid)("errc=%d", &errc) defer log.Trace(path, "uid=%d, gid=%d", uid, gid)("errc=%d", &errc)
fsys.rateLimit()
// This is a no-op for rclone // This is a no-op for rclone
return 0 return 0
} }
@@ -532,6 +571,7 @@ func (fsys *FS) Chown(path string, uid uint32, gid uint32) (errc int) {
// Access checks file access permissions. // Access checks file access permissions.
func (fsys *FS) Access(path string, mask uint32) (errc int) { func (fsys *FS) Access(path string, mask uint32) (errc int) {
defer log.Trace(path, "mask=0%o", mask)("errc=%d", &errc) defer log.Trace(path, "mask=0%o", mask)("errc=%d", &errc)
fsys.rateLimit()
// This is a no-op for rclone // This is a no-op for rclone
return 0 return 0
} }
@@ -539,27 +579,32 @@ func (fsys *FS) Access(path string, mask uint32) (errc int) {
// Fsyncdir synchronizes directory contents. // Fsyncdir synchronizes directory contents.
func (fsys *FS) Fsyncdir(path string, datasync bool, fh uint64) (errc int) { func (fsys *FS) Fsyncdir(path string, datasync bool, fh uint64) (errc int) {
defer log.Trace(path, "datasync=%v, fh=0x%X", datasync, fh)("errc=%d", &errc) defer log.Trace(path, "datasync=%v, fh=0x%X", datasync, fh)("errc=%d", &errc)
fsys.rateLimit()
// This is a no-op for rclone // This is a no-op for rclone
return 0 return 0
} }
// Setxattr sets extended attributes. // Setxattr sets extended attributes.
func (fsys *FS) Setxattr(path string, name string, value []byte, flags int) (errc int) { func (fsys *FS) Setxattr(path string, name string, value []byte, flags int) (errc int) {
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }
// Getxattr gets extended attributes. // Getxattr gets extended attributes.
func (fsys *FS) Getxattr(path string, name string) (errc int, value []byte) { func (fsys *FS) Getxattr(path string, name string) (errc int, value []byte) {
fsys.rateLimit()
return -fuse.ENOSYS, nil return -fuse.ENOSYS, nil
} }
// Removexattr removes extended attributes. // Removexattr removes extended attributes.
func (fsys *FS) Removexattr(path string, name string) (errc int) { func (fsys *FS) Removexattr(path string, name string) (errc int) {
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }
// Listxattr lists extended attributes. // Listxattr lists extended attributes.
func (fsys *FS) Listxattr(path string, fill func(name string) bool) (errc int) { func (fsys *FS) Listxattr(path string, fill func(name string) bool) (errc int) {
fsys.rateLimit()
return -fuse.ENOSYS return -fuse.ENOSYS
} }

View File

@@ -138,7 +138,7 @@ func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error,
// Create underlying FS // Create underlying FS
f := VFS.Fs() f := VFS.Fs()
fsys := NewFS(VFS) fsys := NewFS(VFS, opt)
host := fuse.NewFileSystemHost(fsys) host := fuse.NewFileSystemHost(fsys)
host.SetCapReaddirPlus(true) // only works on Windows host.SetCapReaddirPlus(true) // only works on Windows
host.SetCapCaseInsensitive(f.Features().CaseInsensitive) host.SetCapCaseInsensitive(f.Features().CaseInsensitive)

View File

@@ -45,6 +45,7 @@ type Options struct {
DaemonTimeout time.Duration // OSXFUSE only DaemonTimeout time.Duration // OSXFUSE only
AsyncRead bool AsyncRead bool
NetworkMode bool // Windows only NetworkMode bool // Windows only
TPSLimit float64
} }
// DefaultOpt is the default values for creating the mount // DefaultOpt is the default values for creating the mount
@@ -97,6 +98,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.BoolVarP(flagSet, &Opt.AsyncRead, "async-read", "", Opt.AsyncRead, "Use asynchronous reads. Not supported on Windows.") flags.BoolVarP(flagSet, &Opt.AsyncRead, "async-read", "", Opt.AsyncRead, "Use asynchronous reads. Not supported on Windows.")
flags.FVarP(flagSet, &Opt.MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads. Not supported on Windows.") flags.FVarP(flagSet, &Opt.MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads. Not supported on Windows.")
flags.BoolVarP(flagSet, &Opt.WritebackCache, "write-back-cache", "", Opt.WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. Not supported on Windows.") flags.BoolVarP(flagSet, &Opt.WritebackCache, "write-back-cache", "", Opt.WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used. Not supported on Windows.")
flags.Float64VarP(flagSet, &Opt.TPSLimit, "mount-tpslimit", "", 0, "Limit mount transactions per second to this. (0 is off).")
// Windows and OSX // Windows and OSX
flags.StringVarP(flagSet, &Opt.VolumeName, "volname", "", Opt.VolumeName, "Set the volume name. Supported on Windows and OSX only.") flags.StringVarP(flagSet, &Opt.VolumeName, "volname", "", Opt.VolumeName, "Set the volume name. Supported on Windows and OSX only.")
// OSX only // OSX only

View File

@@ -456,3 +456,4 @@ put them back in again.` >}}
* Nicolas Rueff <nicolas@rueff.fr> * Nicolas Rueff <nicolas@rueff.fr>
* Pau Rodriguez-Estivill <prodrigestivill@gmail.com> * Pau Rodriguez-Estivill <prodrigestivill@gmail.com>
* Bob Pusateri <BobPusateri@users.noreply.github.com> * Bob Pusateri <BobPusateri@users.noreply.github.com>
* Alex JOST <25005220+dimejo@users.noreply.github.com>

View File

@@ -72,7 +72,7 @@ If your browser doesn't open automatically go to the following link: http://127.
Log in and authorize rclone for access Log in and authorize rclone for access
Waiting for code... Waiting for code...
Got code Got code
Configure this as a team drive? Configure this as a Shared Drive (Team Drive)?
y) Yes y) Yes
n) No n) No
y/n> n y/n> n
@@ -279,23 +279,24 @@ Note: in case you configured a specific root folder on gdrive and rclone is unab
`rclone -v foo@example.com lsf gdrive:backup` `rclone -v foo@example.com lsf gdrive:backup`
### Team drives ### ### Shared drives (team drives) ###
If you want to configure the remote to point to a Google Team Drive If you want to configure the remote to point to a Google Shared Drive
then answer `y` to the question `Configure this as a team drive?`. (previously known as Team Drives) then answer `y` to the question
`Configure this as a Shared Drive (Team Drive)?`.
This will fetch the list of Team Drives from google and allow you to This will fetch the list of Shared Drives from google and allow you to
configure which one you want to use. You can also type in a team configure which one you want to use. You can also type in a Shared
drive ID if you prefer. Drive ID if you prefer.
For example: For example:
``` ```
Configure this as a team drive? Configure this as a Shared Drive (Team Drive)?
y) Yes y) Yes
n) No n) No
y/n> y y/n> y
Fetching team drive list... Fetching Shared Drive list...
Choose a number from below, or type in your own value Choose a number from below, or type in your own value
1 / Rclone Test 1 / Rclone Test
\ "xxxxxxxxxxxxxxxxxxxx" \ "xxxxxxxxxxxxxxxxxxxx"
@@ -303,7 +304,7 @@ Choose a number from below, or type in your own value
\ "yyyyyyyyyyyyyyyyyyyy" \ "yyyyyyyyyyyyyyyyyyyy"
3 / Rclone Test 3 3 / Rclone Test 3
\ "zzzzzzzzzzzzzzzzzzzz" \ "zzzzzzzzzzzzzzzzzzzz"
Enter a Team Drive ID> 1 Enter a Shared Drive ID> 1
-------------------- --------------------
[remote] [remote]
client_id = client_id =
@@ -674,7 +675,7 @@ Needed only if you want use SA instead of interactive login.
#### --drive-team-drive #### --drive-team-drive
ID of the Team Drive ID of the Shared Drive (Team Drive)
- Config: team_drive - Config: team_drive
- Env Var: RCLONE_DRIVE_TEAM_DRIVE - Env Var: RCLONE_DRIVE_TEAM_DRIVE
@@ -1137,11 +1138,11 @@ Options:
#### drives #### drives
List the shared drives available to this account List the Shared Drives available to this account
rclone backend drives remote: [options] [<arguments>+] rclone backend drives remote: [options] [<arguments>+]
This command lists the shared drives (teamdrives) available to this This command lists the Shared Drives (Team Drives) available to this
account. account.
Usage: Usage:

39
lib/cache/cache.go vendored
View File

@@ -22,17 +22,15 @@ func New() *Cache {
return &Cache{ return &Cache{
cache: map[string]*cacheEntry{}, cache: map[string]*cacheEntry{},
expireRunning: false, expireRunning: false,
expireDuration: 24 * time.Hour, expireDuration: 300 * time.Second,
expireInterval: 60 * time.Second, expireInterval: 60 * time.Second,
} }
} }
// cacheEntry is stored in the cache // cacheEntry is stored in the cache
type cacheEntry struct { type cacheEntry struct {
createMu sync.Mutex // held while creating the item
value interface{} // cached item value interface{} // cached item
err error // creation error err error // creation error
ok bool // true if entry is valid
key string // key key string // key
lastUsed time.Time // time used for expiry lastUsed time.Time // time used for expiry
pinCount int // non zero if the entry should not be removed pinCount int // non zero if the entry should not be removed
@@ -57,27 +55,23 @@ func (c *Cache) used(entry *cacheEntry) {
// afresh with the create function. // afresh with the create function.
func (c *Cache) Get(key string, create CreateFunc) (value interface{}, err error) { func (c *Cache) Get(key string, create CreateFunc) (value interface{}, err error) {
c.mu.Lock() c.mu.Lock()
entry, found := c.cache[key] entry, ok := c.cache[key]
if !found { if !ok {
entry = &cacheEntry{ c.mu.Unlock() // Unlock in case Get is called recursively
key: key, value, ok, err = create(key)
if err != nil && !ok {
return value, err
} }
entry = &cacheEntry{
value: value,
key: key,
err: err,
}
c.mu.Lock()
c.cache[key] = entry c.cache[key] = entry
} }
c.mu.Unlock() defer c.mu.Unlock()
// Only one racing Get will have found=false here c.used(entry)
entry.createMu.Lock()
if !found {
entry.value, entry.ok, entry.err = create(key)
}
entry.createMu.Unlock()
c.mu.Lock()
if !found && !entry.ok {
delete(c.cache, key)
} else {
c.used(entry)
}
c.mu.Unlock()
return entry.value, entry.err return entry.value, entry.err
} }
@@ -108,7 +102,6 @@ func (c *Cache) Put(key string, value interface{}) {
entry := &cacheEntry{ entry := &cacheEntry{
value: value, value: value,
key: key, key: key,
ok: true,
} }
c.used(entry) c.used(entry)
c.cache[key] = entry c.cache[key] = entry
@@ -119,7 +112,7 @@ func (c *Cache) GetMaybe(key string) (value interface{}, found bool) {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
entry, found := c.cache[key] entry, found := c.cache[key]
if !found || !entry.ok { if !found {
return nil, found return nil, found
} }
c.used(entry) c.used(entry)

View File

@@ -3,8 +3,6 @@ package cache
import ( import (
"errors" "errors"
"fmt" "fmt"
"sync"
"sync/atomic"
"testing" "testing"
"time" "time"
@@ -13,7 +11,7 @@ import (
) )
var ( var (
called = int32(0) called = 0
errSentinel = errors.New("an error") errSentinel = errors.New("an error")
errCached = errors.New("a cached error") errCached = errors.New("a cached error")
) )
@@ -21,19 +19,17 @@ var (
func setup(t *testing.T) (*Cache, CreateFunc) { func setup(t *testing.T) (*Cache, CreateFunc) {
called = 0 called = 0
create := func(path string) (interface{}, bool, error) { create := func(path string) (interface{}, bool, error) {
newCalled := atomic.AddInt32(&called, 1) assert.Equal(t, 0, called)
assert.Equal(t, int32(1), newCalled) called++
switch path { switch path {
case "/": case "/":
time.Sleep(100 * time.Millisecond)
return "/", true, nil return "/", true, nil
case "/file.txt": case "/file.txt":
return "/file.txt", true, errCached return "/file.txt", true, errCached
case "/error": case "/error":
return nil, false, errSentinel return nil, false, errSentinel
} }
assert.Fail(t, fmt.Sprintf("Unknown path %q", path)) panic(fmt.Sprintf("Unknown path %q", path))
return nil, false, nil
} }
c := New() c := New()
return c, create return c, create
@@ -55,24 +51,6 @@ func TestGet(t *testing.T) {
assert.Equal(t, f, f2) assert.Equal(t, f, f2)
} }
func TestGetConcurrent(t *testing.T) {
c, create := setup(t)
assert.Equal(t, 0, len(c.cache))
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_, err := c.Get("/", create)
require.NoError(t, err)
}()
}
wg.Wait()
assert.Equal(t, 1, len(c.cache))
}
func TestGetFile(t *testing.T) { func TestGetFile(t *testing.T) {
c, create := setup(t) c, create := setup(t)