fshttp: add --dump curl for dumping HTTP requests as curl commands

Previously if auth keys were provided without a comma then rclone
would only log an INFO message which could mean it went on to serve
without any auth.

The parsing for environment variables was changed in v1.70.0 to make
them work properly with multiple inputs. This means the input is
treated like a mini CSV file which works well except in this case when
the input has commas. This meant `user,auth` without quotes is treated
as two key pairs `user` and `quote`. The correct syntax is
`"user,auth"`. This updates the documentation accordingly.

backend/memory/memory.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -25,6 +26,7 @@ var (
 	hashType = hash.MD5
 	// the object storage is persistent
 	buckets      = newBucketsInfo()
+	errWriteOnly = errors.New("can't read when using --memory-discard")
 )
 
 // Register with Fs
@@ -33,12 +35,32 @@ func init() {
 		Name:        "memory",
 		Description: "In memory object storage system.",
 		NewFs:       NewFs,
-		Options:     []fs.Option{},
+		Options: []fs.Option{{
+			Name:     "discard",
+			Default:  false,
+			Advanced: true,
+			Help: `If set all writes will be discarded and reads will return an error
+
+If set then when files are uploaded the contents not be saved. The
+files will appear to have been uploaded but will give an error on
+read. Files will have their MD5 sum calculated on upload which takes
+very little CPU time and allows the transfers to be checked.
+
+This can be useful for testing performance.
+
+Probably most easily used by using the connection string syntax:
+
+    :memory,discard:bucket
+
+`,
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
-type Options struct{}
+type Options struct {
+	Discard bool `config:"discard"`
+}
 
 // Fs represents a remote memory server
 type Fs struct {
@@ -164,6 +186,7 @@ type objectData struct {
 	hash     string
 	mimeType string
 	data     []byte
+	size     int64
 }
 
 // Object describes a memory object
@@ -558,7 +581,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hashType {
 		return "", hash.ErrUnsupported
 	}
-	if o.od.hash == "" {
+	if o.od.hash == "" && !o.fs.opt.Discard {
 		sum := md5.Sum(o.od.data)
 		o.od.hash = hex.EncodeToString(sum[:])
 	}
@@ -567,7 +590,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	return int64(len(o.od.data))
+	return o.od.size
 }
 
 // ModTime returns the modification time of the object
@@ -593,6 +616,9 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	if o.fs.opt.Discard {
+		return nil, errWriteOnly
+	}
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
 		switch x := option.(type) {
@@ -624,13 +650,24 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := io.ReadAll(in)
+	var data []byte
+	var size int64
+	var hash string
+	if o.fs.opt.Discard {
+		h := md5.New()
+		size, err = io.Copy(h, in)
+		hash = hex.EncodeToString(h.Sum(nil))
+	} else {
+		data, err = io.ReadAll(in)
+		size = int64(len(data))
+	}
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}
 	o.od = &objectData{
 		data:     data,
-		hash:     "",
+		size:     size,
+		hash:     hash,
 		modTime:  src.ModTime(ctx),
 		mimeType: fs.MimeType(ctx, src),
 	}

backend/pcloud/api/types.go

@@ -222,3 +222,11 @@ type UserInfo struct {
 		} `json:"steps"`
 	} `json:"journey"`
 }
+
+// DiffResult is the response from /diff
+type DiffResult struct {
+	Result  int              `json:"result"`
+	DiffID  int64            `json:"diffid"`
+	Entries []map[string]any `json:"entries"`
+	Error   string           `json:"error"`
+}

backend/pcloud/pcloud.go

@@ -171,6 +171,7 @@ type Fs struct {
 	dirCache     *dircache.DirCache     // Map of directory path to directory id
 	pacer        *fs.Pacer              // pacer for API calls
 	tokenRenewer *oauthutil.Renew       // renew the token on expiry
+	lastDiffID   int64                  // change tracking state for diff long-polling
 }
 
 // Object describes a pcloud object
@@ -1033,6 +1034,137 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+// ChangeNotify implements fs.Features.ChangeNotify
+func (f *Fs) ChangeNotify(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
+	// Start long-poll loop in background
+	go f.changeNotifyLoop(ctx, notify, ch)
+}
+
+// changeNotifyLoop contains the blocking long-poll logic.
+func (f *Fs) changeNotifyLoop(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
+	// Standard polling interval
+	interval := 30 * time.Second
+
+	// Start with diffID = 0 to get the current state
+	var diffID int64
+
+	// Helper to process changes from the diff API
+	handleChanges := func(entries []map[string]any) {
+		notifiedPaths := make(map[string]bool)
+
+		for _, entry := range entries {
+			meta, ok := entry["metadata"].(map[string]any)
+			if !ok {
+				continue
+			}
+
+			// Robust extraction of ParentFolderID
+			var pid int64
+			if val, ok := meta["parentfolderid"]; ok {
+				switch v := val.(type) {
+				case float64:
+					pid = int64(v)
+				case int64:
+					pid = v
+				case int:
+					pid = int64(v)
+				}
+			}
+
+			// Resolve the path using dirCache.GetInv
+			// pCloud uses "d" prefix for directory IDs in cache, but API returns numbers
+			dirID := fmt.Sprintf("d%d", pid)
+			parentPath, ok := f.dirCache.GetInv(dirID)
+
+			if !ok {
+				// Parent not in cache, so we can ignore this change as it is outside
+				// of what the mount has seen or cares about.
+				continue
+			}
+
+			name, _ := meta["name"].(string)
+			fullPath := path.Join(parentPath, name)
+
+			// Determine EntryType (File or Directory)
+			entryType := fs.EntryObject
+			if isFolder, ok := meta["isfolder"].(bool); ok && isFolder {
+				entryType = fs.EntryDirectory
+			}
+
+			// Deduplicate notifications for this batch
+			if !notifiedPaths[fullPath] {
+				fs.Debugf(f, "ChangeNotify: detected change in %q (type: %v)", fullPath, entryType)
+				notify(fullPath, entryType)
+				notifiedPaths[fullPath] = true
+			}
+		}
+	}
+
+	for {
+		// Check context and channel
+		select {
+		case <-ctx.Done():
+			return
+		case newInterval, ok := <-ch:
+			if !ok {
+				return
+			}
+			interval = newInterval
+		default:
+		}
+
+		// Setup /diff Request
+		opts := rest.Opts{
+			Method:     "GET",
+			Path:       "/diff",
+			Parameters: url.Values{},
+		}
+
+		if diffID != 0 {
+			opts.Parameters.Set("diffid", strconv.FormatInt(diffID, 10))
+			opts.Parameters.Set("block", "1")
+		} else {
+			opts.Parameters.Set("last", "0")
+		}
+
+		// Perform Long-Poll
+		// Timeout set to 90s (server usually blocks for 60s max)
+		reqCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
+		var result api.DiffResult
+
+		_, err := f.srv.CallJSON(reqCtx, &opts, nil, &result)
+		cancel()
+
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			// Ignore timeout errors as they are normal for long-polling
+			if !errors.Is(err, context.DeadlineExceeded) {
+				fs.Infof(f, "ChangeNotify: polling error: %v. Waiting %v.", err, interval)
+				time.Sleep(interval)
+			}
+			continue
+		}
+
+		// If result is not 0, reset DiffID to resync
+		if result.Result != 0 {
+			diffID = 0
+			time.Sleep(2 * time.Second)
+			continue
+		}
+
+		if result.DiffID != 0 {
+			diffID = result.DiffID
+			f.lastDiffID = diffID
+		}
+
+		if len(result.Entries) > 0 {
+			handleChanges(result.Entries)
+		}
+	}
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	// EU region supports SHA1 and SHA256 (but rclone doesn't
@@ -1401,6 +1533,7 @@ var (
 	_ fs.ListPer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/shade/shade.go

@@ -311,6 +311,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	o := &Object{
 		fs:     f,
 		remote: remote,
+		mtime:  srcObj.mtime,
+		size:   srcObj.size,
 	}
 	fromFullPath := path.Join(src.Fs().Root(), srcObj.remote)
 	toFullPath := path.Join(f.root, remote)
@@ -367,7 +369,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	err := f.ensureParentDirectories(ctx, dstRemote)
+	fullPathSrc := f.buildFullPath(srcRemote)
+	fullPathSrcUnencoded, err := url.QueryUnescape(fullPathSrc)
+	if err != nil {
+		return err
+	}
+
+	fullPathDstUnencoded, err := url.QueryUnescape(fullPath)
+	if err != nil {
+		return err
+	}
+
+	err = f.ensureParentDirectories(ctx, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -378,6 +391,15 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	_, err = f.Move(ctx, o, dstRemote)
+
+	if err == nil {
+
+		f.createdDirMu.Lock()
+		f.createdDirs[fullPathSrcUnencoded] = false
+		f.createdDirs[fullPathDstUnencoded] = true
+		f.createdDirMu.Unlock()
+	}
+
 	return err
 }
 

cmd/ls/lshelp/lshelp.go

@@ -26,6 +26,10 @@ Note that |ls| and |lsl| recurse by default - use |--max-depth 1| to stop the re
 The other list commands |lsd|,|lsf|,|lsjson| do not recurse by default -
 use |-R| to make them recurse.
 
+List commands prefer a recursive method that uses more memory but fewer
+transactions by default. Use |--disable ListR| to suppress the behavior.
+See [|--fast-list|](/docs/#fast-list) for more details.
+
 Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).`, "|", "`")

cmd/serve/s3/serve_s3.md

@@ -13,6 +13,26 @@ docs](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)).
 `--auth-key` is not provided then `serve s3` will allow anonymous
 access.
 
+Like all rclone flags `--auth-key` can be set via environment
+variables, in this case `RCLONE_AUTH_KEY`. Since this flag can be
+repeated, the input to `RCLONE_AUTH_KEY` is CSV encoded. Because the
+`accessKey,secretKey` has a comma in, this means it needs to be in
+quotes.
+
+```console
+export RCLONE_AUTH_KEY='"user,pass"'
+rclone serve s3 ...
+```
+
+Or to supply multiple identities:
+
+```console
+export RCLONE_AUTH_KEY='"user1,pass1","user2,pass2"'
+rclone serve s3 ...
+```
+
+Setting this variable without quotes will produce an error.
+
 Please note that some clients may require HTTPS endpoints. See [the
 SSL docs](#tls-ssl) for more information.
 

cmd/serve/s3/server.go

@@ -70,6 +70,11 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 		w.s3Secret = getAuthSecret(opt.AuthKey)
 	}
 
+	authList, err := authlistResolver(opt.AuthKey)
+	if err != nil {
+		return nil, fmt.Errorf("parsing auth list failed: %q", err)
+	}
+
 	var newLogger logger
 	w.faker = gofakes3.New(
 		newBackend(w),
@@ -77,7 +82,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 		gofakes3.WithLogger(newLogger),
 		gofakes3.WithRequestID(rand.Uint64()),
 		gofakes3.WithoutVersioning(),
-		gofakes3.WithV4Auth(authlistResolver(opt.AuthKey)),
+		gofakes3.WithV4Auth(authList),
 		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
 	)
 
@@ -92,7 +97,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 		w._vfs = vfs.New(f, vfsOpt)
 
 		if len(opt.AuthKey) > 0 {
-			w.faker.AddAuthKeys(authlistResolver(opt.AuthKey))
+			w.faker.AddAuthKeys(authList)
 		}
 	}
 

cmd/serve/s3/utils.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"io"
 	"os"
 	"path"
@@ -125,15 +126,14 @@ func rmdirRecursive(p string, VFS *vfs.VFS) {
 	}
 }
 
-func authlistResolver(list []string) map[string]string {
+func authlistResolver(list []string) (map[string]string, error) {
 	authList := make(map[string]string)
 	for _, v := range list {
 		parts := strings.Split(v, ",")
 		if len(parts) != 2 {
-			fs.Infof(nil, "Ignored: invalid auth pair %s", v)
+			return nil, errors.New("invalid auth pair: expecting a single comma")
-			continue
 		}
 		authList[parts[0]] = parts[1]
 	}
-	return authList
+	return authList, nil
 }

docs/content/authors.md

@@ -1058,3 +1058,5 @@ put them back in again. -->
 - Tingsong Xu <tingsong.xu@rightcapital.com>
 - Jonas Tingeborn <134889+jojje@users.noreply.github.com>
 - jhasse-shade <jacob@shade.inc>
+- vyv03354 <VYV03354@nifty.ne.jp>
+- masrlinu <masrlinu@users.noreply.github.com> <5259918+masrlinu@users.noreply.github.com>

docs/content/docs.md

@@ -3278,6 +3278,10 @@ The available flags are:
 - `mapper` dumps the JSON blobs being sent to the program supplied with
   `--metadata-mapper` and received from it. It can be useful for debugging
   the metadata mapper interface.
+- `curl` dumps the HTTP request as a `curl` command. Can be used with
+  the other HTTP debugging flags (e.g. `requests`, `bodies`). By
+  default the auth will be masked - use with `auth` to have the curl
+  commands with authentication too.
 
 ## Filtering
 

docs/content/pcloud.md

@@ -173,6 +173,31 @@ So if the folder you want rclone to use your is "My Music/", then use the return
 id from ```rclone lsf``` command (ex. `dxxxxxxxx2`) as the `root_folder_id` variable
 value in the config file.
 
+### Change notifications and mounts
+
+The pCloud backend supports real‑time updates for rclone mounts via change
+notifications. rclone uses pCloud’s diff long‑polling API to detect changes and
+will automatically refresh directory listings in the mounted filesystem when
+changes occur.
+
+Notes and behavior:
+
+- Works automatically when using `rclone mount` and requires no additional
+  configuration.
+- Notifications are directory‑scoped: when rclone detects a change, it refreshes
+  the affected directory so new/removed/renamed files become visible promptly.
+- Updates are near real‑time. The backend uses a long‑poll with short fallback
+  polling intervals, so you should see changes appear quickly without manual
+  refreshes.
+
+If you want to debug or verify notifications, you can use the helper command:
+
+```bash
+rclone test changenotify remote:
+```
+
+This will log incoming change notifications for the given remote.
+
 <!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/pcloud/pcloud.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
 ### Standard options
 

fs/dump.go

@@ -14,6 +14,7 @@ const (
 	DumpGoRoutines
 	DumpOpenFiles
 	DumpMapper
+	DumpCurl
 )
 
 type dumpChoices struct{}
@@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {
 		{uint64(DumpGoRoutines), "goroutines"},
 		{uint64(DumpOpenFiles), "openfiles"},
 		{uint64(DumpMapper), "mapper"},
+		{uint64(DumpCurl), "curl"},
 	}
 }
 

fs/fshttp/http.go

@@ -15,6 +15,8 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -24,6 +26,7 @@ import (
 	"github.com/rclone/rclone/lib/structs"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/net/publicsuffix"
+	"moul.io/http2curl/v2"
 )
 
 const (
@@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte {
 	return buf
 }
 
+// cleanCurl gets rid of Auth headers in a curl command
+func cleanCurl(cmd *http2curl.CurlCommand) {
+	for _, authBuf := range authBufs {
+		auth := "'" + string(authBuf)
+		for i, arg := range *cmd {
+			if strings.HasPrefix(arg, auth) {
+				(*cmd)[i] = auth + "XXXX'"
+			}
+		}
+	}
+}
+
 var expireWindow = 30 * time.Second
 
 func isCertificateExpired(cc *tls.Config) bool {
@@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorReq)
 		logMutex.Unlock()
 	}
+	// Dump curl request
+	if t.dump&(fs.DumpCurl) != 0 {
+		cmd, err := http2curl.GetCurlCommand(req)
+		if err != nil {
+			fs.Debugf(nil, "Failed to create curl command: %v", err)
+		} else {
+			// Patch -X HEAD into --head
+			for i := range len(*cmd) - 1 {
+				if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" {
+					(*cmd)[i] = "--head"
+					*cmd = slices.Delete(*cmd, i+1, i+2)
+					break
+				}
+			}
+			if t.dump&fs.DumpAuth == 0 {
+				cleanCurl(cmd)
+			}
+			fs.Debugf(nil, "HTTP REQUEST: %v", cmd)
+		}
+	}
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response

fs/fshttp/http_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"moul.io/http2curl/v2"
 )
 
 func TestCleanAuth(t *testing.T) {
@@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) {
 	}
 }
 
+func TestCleanCurl(t *testing.T) {
+	for _, test := range []struct {
+		in   []string
+		want []string
+	}{{
+		[]string{""},
+		[]string{""},
+	}, {
+		[]string{"floo"},
+		[]string{"floo"},
+	}, {
+		[]string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'Authorization: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""},
+	}} {
+		in := http2curl.CurlCommand(test.in)
+		cleanCurl(&in)
+		assert.Equal(t, test.want, test.in, test.in)
+	}
+}
+
 var certSerial = int64(0)
 
 // Create a test certificate and key pair that is valid for a specific