vfs: factor the vfs cache into its own package

vfs/read_write.go

@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/log"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/file"
 )
 
@@ -55,12 +54,12 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
 	}
 
 	// mark the file as open in the cache - must be done before the mkdir
-	fh.d.VFS().cache.open(fh.file.Path())
+	fh.d.VFS().cache.Open(fh.file.Path())
 
 	// Make a place for the file
-	_, err = d.VFS().cache.mkdir(fh.file.Path())
+	_, err = d.VFS().cache.Mkdir(fh.file.Path())
 	if err != nil {
-		fh.d.VFS().cache.close(fh.file.Path())
+		fh.d.VFS().cache.Close(fh.file.Path())
 		return nil, errors.Wrap(err, "open RW handle failed to make cache directory")
 	}
 
@@ -80,16 +79,6 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
 	return fh, nil
 }
 
-// copy an object to or from the remote while accounting for it
-func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
-	if operations.NeedTransfer(context.TODO(), dst, src) {
-		newDst, err = operations.Copy(context.TODO(), f, dst, remote, src)
-	} else {
-		newDst = dst
-	}
-	return newDst, err
-}
-
 // openPending opens the file if there is a pending open
 //
 // call with the lock held
@@ -110,12 +99,9 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
 		// If the remote object exists AND its cached file exists locally AND there are no
 		// other RW handles with it open, then attempt to update it.
 		if o != nil && fh.file.rwOpens() == 0 {
-			cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path())
-			if err == nil && cacheObj != nil {
-				_, err = copyObj(fh.d.VFS().cache.f, cacheObj, fh.file.Path(), o)
-				if err != nil {
-					return errors.Wrap(err, "open RW handle failed to update cached file")
-				}
+			err = fh.d.VFS().cache.Check(context.TODO(), o, fh.file.Path())
+			if err != nil {
+				return errors.Wrap(err, "open RW handle failed to check cache file")
 			}
 		}
 
@@ -125,7 +111,7 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
 			// cache file does not exist, so need to fetch it if we have an object to fetch
 			// it from
 			if o != nil {
-				_, err = copyObj(fh.d.VFS().cache.f, nil, fh.file.Path(), o)
+				err = fh.d.VFS().cache.Fetch(context.TODO(), o, fh.file.Path())
 				if err != nil {
 					cause := errors.Cause(err)
 					if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound {
@@ -276,22 +262,8 @@ func (fh *RWFileHandle) flushWrites(closeFile bool) error {
 	}
 
 	if isCopied {
-		// Transfer the temp file to the remote
-		cacheObj, err := fh.d.VFS().cache.f.NewObject(context.TODO(), fh.file.Path())
+		o, err := fh.d.VFS().cache.Store(context.TODO(), fh.file.getObject(), fh.file.Path())
 		if err != nil {
-			err = errors.Wrap(err, "failed to find cache file")
-			fs.Errorf(fh.logPrefix(), "%v", err)
-			return err
-		}
-
-		objPath := fh.file.Path()
-		objOld := fh.file.getObject()
-		if objOld != nil {
-			objPath = objOld.Remote() // use the path of the actual object if available
-		}
-		o, err := copyObj(fh.d.VFS().f, objOld, objPath, cacheObj)
-		if err != nil {
-			err = errors.Wrap(err, "failed to transfer file from cache to remote")
 			fs.Errorf(fh.logPrefix(), "%v", err)
 			return err
 		}
@@ -322,7 +294,7 @@ func (fh *RWFileHandle) close() (err error) {
 		if fh.opened {
 			fh.file.delRWOpen()
 		}
-		fh.d.VFS().cache.close(fh.file.Path())
+		fh.d.VFS().cache.Close(fh.file.Path())
 	}()
 
 	return fh.flushWrites(true)
@@ -501,6 +473,10 @@ func (fh *RWFileHandle) Write(b []byte) (n int, err error) {
 
 // WriteAt bytes to the file at off
 func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) {
+	if fh.flags&os.O_APPEND != 0 {
+		// if append is set, call Write as WriteAt returns an error if append is set
+		return fh.Write(b)
+	}
 	err = fh.writeFn(func() error {
 		n, err = fh.File.WriteAt(b, off)
 		return err