Added support for Max Pages

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

cmd/mount/mount.go

@@ -32,12 +32,14 @@ func mountOptions(device string) (options []fuse.MountOption) {
 		fuse.Subtype("rclone"),
 		fuse.FSName(device),
 		fuse.VolumeName(mountlib.VolumeName),
-		fuse.AsyncRead(),
 
 		// Options from benchmarking in the fuse module
 		//fuse.MaxReadahead(64 * 1024 * 1024),
 		//fuse.WritebackCache(),
 	}
+	if mountlib.AsyncRead {
+		options = append(options, fuse.AsyncRead())
+	}
 	if mountlib.NoAppleDouble {
 		options = append(options, fuse.NoAppleDouble())
 	}
@@ -71,6 +73,9 @@ func mountOptions(device string) (options []fuse.MountOption) {
 	if len(mountlib.ExtraFlags) > 0 {
 		fs.Errorf(nil, "--fuse-flag not supported with this FUSE backend")
 	}
+	if vfsflags.Opt.MaxPages != 0 {
+		options = append(options, fuse.MaxPages(vfsflags.Opt.MaxPages))
+	}
 	return options
 }
 

cmd/mountlib/mount.go

@@ -36,6 +36,7 @@ var (
 	NoAppleDouble      = true        // use noappledouble by default
 	NoAppleXattr       = false       // do not use noapplexattr by default
 	DaemonTimeout      time.Duration // OSXFUSE only
+	AsyncRead          = true        // do async reads by default
 )
 
 // Global constants
@@ -318,6 +319,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
 	flags.BoolVarP(cmdFlags, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
 	flags.StringVarP(cmdFlags, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
 	flags.DurationVarP(cmdFlags, &DaemonTimeout, "daemon-timeout", "", DaemonTimeout, "Time limit for rclone to respond to kernel (not supported by all OSes).")
+	flags.BoolVarP(cmdFlags, &AsyncRead, "async-read", "", AsyncRead, "Use asynchronous reads.")
 
 	if runtime.GOOS == "darwin" {
 		flags.BoolVarP(cmdFlags, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")

fs/config/flags/flags.go

@@ -98,6 +98,14 @@ func IntVarP(flags *pflag.FlagSet, p *int, name, shorthand string, value int, us
 	setDefaultFromEnv(flags, name)
 }
 
+// Uint16VarP defines a flag which can be overridden by an environment variable
+//
+// It is a thin wrapper around pflag.Uint16VarP
+func Uint16VarP(flags *pflag.FlagSet, p *uint16, name, shorthand string, value uint16, usage string) {
+	flags.Uint16VarP(p, name, shorthand, value, usage)
+	setDefaultFromEnv(flags, name)
+}
+
 // Uint32VarP defines a flag which can be overridden by an environment variable
 //
 // It is a thin wrapper around pflag.Uint32VarP

vendor/bazil.org/fuse/fuse.go

@@ -156,7 +156,8 @@ func (e *MountpointDoesNotExistError) Error() string {
 // progress.
 func Mount(dir string, options ...MountOption) (*Conn, error) {
 	conf := mountConfig{
-		options: make(map[string]string),
+		options:  make(map[string]string),
+		maxPages: 32,
 	}
 	for _, option := range options {
 		if err := option(&conf); err != nil {
@@ -238,10 +239,20 @@ func initMount(c *Conn, conf *mountConfig) error {
 		MaxWrite:     maxWrite,
 		Flags:        InitBigWrites | conf.initFlags,
 	}
+
+	setMaxPages(r, s, conf)
+
 	r.Respond(s)
 	return nil
 }
 
+func setMaxPages(request *InitRequest, response *InitResponse, conf *mountConfig) {
+	if (request.Flags & InitMaxPages) == InitMaxPages {
+		response.Flags |= InitMaxPages
+		response.MaxPages = conf.maxPages
+	}
+}
+
 // A Request represents a single FUSE request received from the kernel.
 // Use a type switch to determine the specific kind.
 // A request of unrecognized type will have concrete type *Header.
@@ -1229,6 +1240,9 @@ type InitResponse struct {
 	// Maximum size of a single write operation.
 	// Linux enforces a minimum of 4 KiB.
 	MaxWrite uint32
+	// Maximum number of pages in a single write operation.
+	// Linux enforces a minimum of 32.
+	MaxPages uint16
 }
 
 func (r *InitResponse) String() string {
@@ -1244,12 +1258,14 @@ func (r *InitRequest) Respond(resp *InitResponse) {
 	out.MaxReadahead = resp.MaxReadahead
 	out.Flags = uint32(resp.Flags)
 	out.MaxWrite = resp.MaxWrite
+	out.MaxPages = resp.MaxPages
 
 	// MaxWrite larger than our receive buffer would just lead to
 	// errors on large writes.
 	if out.MaxWrite > maxWrite {
 		out.MaxWrite = maxWrite
 	}
+
 	r.respond(buf)
 }
 

vendor/bazil.org/fuse/fuse_kernel.go

@@ -274,6 +274,8 @@ const (
 	InitWritebackCache  InitFlags = 1 << 16
 	InitNoOpenSupport   InitFlags = 1 << 17
 
+	InitMaxPages InitFlags = 1 << 22 // Linux only
+
 	InitCaseSensitive InitFlags = 1 << 29 // OS X only
 	InitVolRename     InitFlags = 1 << 30 // OS X only
 	InitXtimes        InitFlags = 1 << 31 // OS X only
@@ -710,6 +712,8 @@ type initOut struct {
 	Flags        uint32
 	Unused       uint32
 	MaxWrite     uint32
+	_            uint32 // Unused, refers to TimeGran
+	MaxPages     uint16
 }
 
 type interruptIn struct {

vendor/bazil.org/fuse/fuse_linux.go

@@ -4,4 +4,6 @@ package fuse
 //
 // Linux 4.2.0 has been observed to cap this value at 128kB
 // (FUSE_MAX_PAGES_PER_REQ=32, 4kB pages).
-const maxWrite = 128 * 1024
+// From Linux 4.20, the cap has been increased to 1MiB
+// (FUSE_MAX_PAGES_PER_REQ=256, 4kB pages).
+const maxWrite = 1 * 1024 * 1024 // 1 MiB

vendor/bazil.org/fuse/options.go

@@ -16,6 +16,7 @@ type mountConfig struct {
 	maxReadahead     uint32
 	initFlags        InitFlags
 	osxfuseLocations []OSXFUSEPaths
+	maxPages         uint16
 }
 
 func escapeComma(s string) string {
@@ -317,3 +318,11 @@ func AllowNonEmptyMount() MountOption {
 		return nil
 	}
 }
+
+// MaxPages enables the configuration of the maximum number of pages
+// in the request & response from the kernel.
+//
+// Linux only. Others ignore this option.
+func MaxPages(count uint16) MountOption {
+	return maxPages(count)
+}

vendor/bazil.org/fuse/options_darwin.go

@@ -38,3 +38,7 @@ func noBrowse(conf *mountConfig) error {
 	conf.options["nobrowse"] = ""
 	return nil
 }
+
+func maxPages(count uint16) MountOption {
+	return dummyOption
+}

vendor/bazil.org/fuse/options_freebsd.go

@@ -30,3 +30,7 @@ func exclCreate(conf *mountConfig) error {
 func noBrowse(conf *mountConfig) error {
 	return nil
 }
+
+func maxPages(count uint16) MountOption {
+	return dummyOption
+}

vendor/bazil.org/fuse/options_linux.go

@@ -27,3 +27,13 @@ func exclCreate(conf *mountConfig) error {
 func noBrowse(conf *mountConfig) error {
 	return nil
 }
+
+func maxPages(count uint16) MountOption {
+	return func(conf *mountConfig) error {
+		if count > 256 || count == 0 {
+			count = 256
+		}
+		conf.maxPages = count
+		return nil
+	}
+}

vfs/read.go

@@ -232,9 +232,10 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
 	if gap := off - fh.offset; gap > 0 && gap < int64(8*maxBuf) {
 		// Set a background timer so we don't wait for long
 		// Waits here potentially affect all seeks so need to keep them short
-		// This time here was made by finding the smallest when mounting a local backend
+		// The default time here was made by finding the
-		// that didn't cause seeks.
+		// smallest when mounting a local backend that didn't
-		const maxWait = 5 * time.Millisecond
+		// cause seeks.
+		maxWait := fh.file.d.vfs.Opt.ReadWait
 		timeout := time.NewTimer(maxWait)
 		done := make(chan struct{})
 		abort := int32(0)

vfs/vfs.go

@@ -53,6 +53,9 @@ var DefaultOpt = Options{
 	ChunkSizeLimit:    -1,
 	CacheMaxSize:      -1,
 	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
+	WriteWait:         1000 * time.Millisecond,
+	ReadWait:          5 * time.Millisecond,
+	MaxPages:          0, // Only for Linux
 }
 
 // Node represents either a directory (*Dir) or a file (*File)
@@ -202,6 +205,9 @@ type Options struct {
 	CacheMaxSize      fs.SizeSuffix
 	CachePollInterval time.Duration
 	CaseInsensitive   bool
+	WriteWait         time.Duration // time to wait for in-sequence write
+	ReadWait          time.Duration // time to wait for in-sequence read
+	MaxPages          uint16
 }
 
 // New creates a new VFS and root directory.  If opt is nil, then

vfs/vfsflags/vfsflags.go

@@ -33,5 +33,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions")
 	flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions")
 	flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match.")
+	flags.DurationVarP(flagSet, &Opt.WriteWait, "vfs-write-wait", "", Opt.WriteWait, "Time to wait for in-sequence write before giving error.")
+	flags.DurationVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", Opt.ReadWait, "Time to wait for in-sequence read before seeking.")
 	platformFlags(flagSet)
 }

vfs/vfsflags/vfsflags_linux.go

@@ -0,0 +1,22 @@
+// +build linux
+
+package vfsflags
+
+import (
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+	"golang.org/x/sys/unix"
+)
+
+// add any extra platform specific flags
+func platformFlags(flagSet *pflag.FlagSet) {
+	flags.IntVarP(flagSet, &Opt.Umask, "umask", "", Opt.Umask, "Override the permission bits set by the filesystem.")
+	Opt.Umask = unix.Umask(0) // read the umask
+	unix.Umask(Opt.Umask)     // set it back to what it was
+	Opt.UID = uint32(unix.Geteuid())
+	Opt.GID = uint32(unix.Getegid())
+	flags.Uint32VarP(flagSet, &Opt.UID, "uid", "", Opt.UID, "Override the uid field set by the filesystem.")
+	flags.Uint32VarP(flagSet, &Opt.GID, "gid", "", Opt.GID, "Override the gid field set by the filesystem.")
+
+	flags.Uint16VarP(flagSet, &Opt.MaxPages, "max-pages", "", Opt.MaxPages, "Set the Max Pages to be passed onto the FUSE library")
+}

vfs/vfsflags/vfsflags_unix.go

@@ -1,4 +1,4 @@
-// +build linux darwin freebsd
+// +build darwin freebsd
 
 package vfsflags
 

vfs/write.go

@@ -132,7 +132,8 @@ func (fh *WriteFileHandle) writeAt(p []byte, off int64) (n int, err error) {
 	}
 	if fh.offset != off {
 		// Set a background timer so we don't wait forever
-		timeout := time.NewTimer(10 * time.Second)
+		maxWait := fh.file.d.vfs.Opt.WriteWait
+		timeout := time.NewTimer(maxWait)
 		done := make(chan struct{})
 		abort := int32(0)
 		go func() {