pcloud: add support for real-time updates in mount

Co-authored-by: masrlinu <5259918+masrlinu@users.noreply.github.com>

backend/memory/memory.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -25,6 +26,7 @@ var (
 	hashType = hash.MD5
 	// the object storage is persistent
 	buckets      = newBucketsInfo()
+	errWriteOnly = errors.New("can't read when using --memory-discard")
 )
 
 // Register with Fs
@@ -33,12 +35,32 @@ func init() {
 		Name:        "memory",
 		Description: "In memory object storage system.",
 		NewFs:       NewFs,
-		Options:     []fs.Option{},
+		Options: []fs.Option{{
+			Name:     "discard",
+			Default:  false,
+			Advanced: true,
+			Help: `If set all writes will be discarded and reads will return an error
+
+If set then when files are uploaded the contents not be saved. The
+files will appear to have been uploaded but will give an error on
+read. Files will have their MD5 sum calculated on upload which takes
+very little CPU time and allows the transfers to be checked.
+
+This can be useful for testing performance.
+
+Probably most easily used by using the connection string syntax:
+
+    :memory,discard:bucket
+
+`,
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
-type Options struct{}
+type Options struct {
+	Discard bool `config:"discard"`
+}
 
 // Fs represents a remote memory server
 type Fs struct {
@@ -164,6 +186,7 @@ type objectData struct {
 	hash     string
 	mimeType string
 	data     []byte
+	size     int64
 }
 
 // Object describes a memory object
@@ -558,7 +581,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hashType {
 		return "", hash.ErrUnsupported
 	}
-	if o.od.hash == "" {
+	if o.od.hash == "" && !o.fs.opt.Discard {
 		sum := md5.Sum(o.od.data)
 		o.od.hash = hex.EncodeToString(sum[:])
 	}
@@ -567,7 +590,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	return int64(len(o.od.data))
+	return o.od.size
 }
 
 // ModTime returns the modification time of the object
@@ -593,6 +616,9 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.fs.opt.Discard {
+		return nil, errWriteOnly
+	}
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
 		switch x := option.(type) {
@@ -624,13 +650,24 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := io.ReadAll(in)
+	var data []byte
+	var size int64
+	var hash string
+	if o.fs.opt.Discard {
+		h := md5.New()
+		size, err = io.Copy(h, in)
+		hash = hex.EncodeToString(h.Sum(nil))
+	} else {
+		data, err = io.ReadAll(in)
+		size = int64(len(data))
+	}
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}
 	o.od = &objectData{
 		data:     data,
-		hash:     "",
+		size:     size,
+		hash:     hash,
 		modTime:  src.ModTime(ctx),
 		mimeType: fs.MimeType(ctx, src),
 	}

backend/pcloud/api/types.go

@@ -222,3 +222,11 @@ type UserInfo struct {
 		} `json:"steps"`
 	} `json:"journey"`
 }
+
+// DiffResult is the response from /diff
+type DiffResult struct {
+	Result  int              `json:"result"`
+	DiffID  int64            `json:"diffid"`
+	Entries []map[string]any `json:"entries"`
+	Error   string           `json:"error"`
+}

backend/pcloud/pcloud.go

@@ -171,6 +171,7 @@ type Fs struct {
 	dirCache     *dircache.DirCache     // Map of directory path to directory id
 	pacer        *fs.Pacer              // pacer for API calls
 	tokenRenewer *oauthutil.Renew       // renew the token on expiry
+	lastDiffID   int64                  // change tracking state for diff long-polling
 }
 
 // Object describes a pcloud object
@@ -1033,6 +1034,137 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+// ChangeNotify implements fs.Features.ChangeNotify
+func (f *Fs) ChangeNotify(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
+	// Start long-poll loop in background
+	go f.changeNotifyLoop(ctx, notify, ch)
+}
+
+// changeNotifyLoop contains the blocking long-poll logic.
+func (f *Fs) changeNotifyLoop(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
+	// Standard polling interval
+	interval := 30 * time.Second
+
+	// Start with diffID = 0 to get the current state
+	var diffID int64
+
+	// Helper to process changes from the diff API
+	handleChanges := func(entries []map[string]any) {
+		notifiedPaths := make(map[string]bool)
+
+		for _, entry := range entries {
+			meta, ok := entry["metadata"].(map[string]any)
+			if !ok {
+				continue
+			}
+
+			// Robust extraction of ParentFolderID
+			var pid int64
+			if val, ok := meta["parentfolderid"]; ok {
+				switch v := val.(type) {
+				case float64:
+					pid = int64(v)
+				case int64:
+					pid = v
+				case int:
+					pid = int64(v)
+				}
+			}
+
+			// Resolve the path using dirCache.GetInv
+			// pCloud uses "d" prefix for directory IDs in cache, but API returns numbers
+			dirID := fmt.Sprintf("d%d", pid)
+			parentPath, ok := f.dirCache.GetInv(dirID)
+
+			if !ok {
+				// Parent not in cache, so we can ignore this change as it is outside
+				// of what the mount has seen or cares about.
+				continue
+			}
+
+			name, _ := meta["name"].(string)
+			fullPath := path.Join(parentPath, name)
+
+			// Determine EntryType (File or Directory)
+			entryType := fs.EntryObject
+			if isFolder, ok := meta["isfolder"].(bool); ok && isFolder {
+				entryType = fs.EntryDirectory
+			}
+
+			// Deduplicate notifications for this batch
+			if !notifiedPaths[fullPath] {
+				fs.Debugf(f, "ChangeNotify: detected change in %q (type: %v)", fullPath, entryType)
+				notify(fullPath, entryType)
+				notifiedPaths[fullPath] = true
+			}
+		}
+	}
+
+	for {
+		// Check context and channel
+		select {
+		case <-ctx.Done():
+			return
+		case newInterval, ok := <-ch:
+			if !ok {
+				return
+			}
+			interval = newInterval
+		default:
+		}
+
+		// Setup /diff Request
+		opts := rest.Opts{
+			Method:     "GET",
+			Path:       "/diff",
+			Parameters: url.Values{},
+		}
+
+		if diffID != 0 {
+			opts.Parameters.Set("diffid", strconv.FormatInt(diffID, 10))
+			opts.Parameters.Set("block", "1")
+		} else {
+			opts.Parameters.Set("last", "0")
+		}
+
+		// Perform Long-Poll
+		// Timeout set to 90s (server usually blocks for 60s max)
+		reqCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
+		var result api.DiffResult
+
+		_, err := f.srv.CallJSON(reqCtx, &opts, nil, &result)
+		cancel()
+
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			// Ignore timeout errors as they are normal for long-polling
+			if !errors.Is(err, context.DeadlineExceeded) {
+				fs.Infof(f, "ChangeNotify: polling error: %v. Waiting %v.", err, interval)
+				time.Sleep(interval)
+			}
+			continue
+		}
+
+		// If result is not 0, reset DiffID to resync
+		if result.Result != 0 {
+			diffID = 0
+			time.Sleep(2 * time.Second)
+			continue
+		}
+
+		if result.DiffID != 0 {
+			diffID = result.DiffID
+			f.lastDiffID = diffID
+		}
+
+		if len(result.Entries) > 0 {
+			handleChanges(result.Entries)
+		}
+	}
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	// EU region supports SHA1 and SHA256 (but rclone doesn't
@@ -1401,6 +1533,7 @@ var (
 	_ fs.ListPer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/shade/shade.go

@@ -311,6 +311,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	o := &Object{
 		fs:     f,
 		remote: remote,
+		mtime:  srcObj.mtime,
+		size:   srcObj.size,
 	}
 	fromFullPath := path.Join(src.Fs().Root(), srcObj.remote)
 	toFullPath := path.Join(f.root, remote)
@@ -367,7 +369,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	err := f.ensureParentDirectories(ctx, dstRemote)
+	fullPathSrc := f.buildFullPath(srcRemote)
+	fullPathSrcUnencoded, err := url.QueryUnescape(fullPathSrc)
+	if err != nil {
+		return err
+	}
+
+	fullPathDstUnencoded, err := url.QueryUnescape(fullPath)
+	if err != nil {
+		return err
+	}
+
+	err = f.ensureParentDirectories(ctx, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -378,6 +391,15 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	_, err = f.Move(ctx, o, dstRemote)
+
+	if err == nil {
+
+		f.createdDirMu.Lock()
+		f.createdDirs[fullPathSrcUnencoded] = false
+		f.createdDirs[fullPathDstUnencoded] = true
+		f.createdDirMu.Unlock()
+	}
+
 	return err
 }
 

cmd/ls/lshelp/lshelp.go

@@ -26,6 +26,10 @@ Note that |ls| and |lsl| recurse by default - use |--max-depth 1| to stop the re
 The other list commands |lsd|,|lsf|,|lsjson| do not recurse by default -
 use |-R| to make them recurse.
 
+List commands prefer a recursive method that uses more memory but fewer
+transactions by default. Use |--disable ListR| to suppress the behavior.
+See [|--fast-list|](/docs/#fast-list) for more details.
+
 Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).`, "|", "`")

docs/content/authors.md

@@ -1058,3 +1058,4 @@ put them back in again. -->
 - Tingsong Xu <tingsong.xu@rightcapital.com>
 - Jonas Tingeborn <134889+jojje@users.noreply.github.com>
 - jhasse-shade <jacob@shade.inc>
+- vyv03354 <VYV03354@nifty.ne.jp>

docs/content/pcloud.md

@@ -173,6 +173,31 @@ So if the folder you want rclone to use your is "My Music/", then use the return
 id from ```rclone lsf``` command (ex. `dxxxxxxxx2`) as the `root_folder_id` variable
 value in the config file.
 
+### Change notifications and mounts
+
+The pCloud backend supports real‑time updates for rclone mounts via change
+notifications. rclone uses pCloud’s diff long‑polling API to detect changes and
+will automatically refresh directory listings in the mounted filesystem when
+changes occur.
+
+Notes and behavior:
+
+- Works automatically when using `rclone mount` and requires no additional
+  configuration.
+- Notifications are directory‑scoped: when rclone detects a change, it refreshes
+  the affected directory so new/removed/renamed files become visible promptly.
+- Updates are near real‑time. The backend uses a long‑poll with short fallback
+  polling intervals, so you should see changes appear quickly without manual
+  refreshes.
+
+If you want to debug or verify notifications, you can use the helper command:
+
+```bash
+rclone test changenotify remote:
+```
+
+This will log incoming change notifications for the given remote.
+
 <!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/pcloud/pcloud.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
 ### Standard options