march: Implement --assume-listings-sorted to speed up sync starting

This can be used when the source and destination backends are
guaranteed to return the items in the same sorted order.

Fixes #5859

README.md

@@ -34,6 +34,7 @@ directories to and from different cloud storage providers.
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
 - Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
 - Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
+- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
 - DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
 - Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
 - Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
@@ -106,6 +107,7 @@ directories to and from different cloud storage providers.
 - Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
 - SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
 - Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
+- Servercore Object Storage [:page_facing_up:](https://rclone.org/s3/#servercore)
 - SFTP [:page_facing_up:](https://rclone.org/sftp/)
 - SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
 - Spectra Logic [:page_facing_up:](https://rclone.org/s3/#spectralogic)

backend/s3/s3.go

@@ -95,6 +95,9 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Cloudflare",
 		Help:  "Cloudflare R2 Storage",
+	}, {
+		Value: "Cubbit",
+		Help:  "Cubbit DS3 Object Storage",
 	}, {
 		Value: "DigitalOcean",
 		Help:  "DigitalOcean Spaces",
@@ -182,6 +185,9 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Selectel",
 		Help:  "Selectel Object Storage",
+	}, {
+		Value: "Servercore",
+		Help:  "Servercore Object Storage",
 	}, {
 		Value: "SpectraLogic",
 		Help:  "Spectra Logic Black Pearl",
@@ -357,6 +363,14 @@ func init() {
 				Value: "auto",
 				Help:  "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Cubbit",
+			Examples: []fs.OptionExample{{
+				Value: "eu-west-1",
+				Help:  "Europe West",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to for FileLu S5.",
@@ -683,6 +697,26 @@ func init() {
 				Value: "ru-1",
 				Help:  "St. Petersburg",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region where your is data stored.\n",
+			Provider: "Servercore",
+			Examples: []fs.OptionExample{{
+				Value: "ru-1",
+				Help:  "St. Petersburg",
+			}, {
+				Value: "gis-1",
+				Help:  "Moscow",
+			}, {
+				Value: "ru-7",
+				Help:  "Moscow",
+			}, {
+				Value: "uz-2",
+				Help:  "Tashkent, Uzbekistan",
+			}, {
+				Value: "kz-1",
+				Help:  "Almaty, Kazakhstan",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region where your data stored.\n",
@@ -714,7 +748,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,FileLu,Hetzner,HuaweiOBS,IDrive,Intercolo,IONOS,Liara,Linode,Magalu,Mega,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,SpectraLogic,Storj,Synology,TencentCOS,Zata",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,Cubbit,FlashBlade,FileLu,Hetzner,HuaweiOBS,IDrive,Intercolo,IONOS,Liara,Linode,Magalu,Mega,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,Servercore,SpectraLogic,Storj,Synology,TencentCOS,Zata",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -915,6 +949,14 @@ func init() {
 				Value: "eos-anhui-1.cmecloud.cn",
 				Help:  "Anhui China (Huainan)",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Cubbit DS3 Object Storage.",
+			Provider: "Cubbit",
+			Examples: []fs.OptionExample{{
+				Value: "s3.cubbit.eu",
+				Help:  "Cubbit DS3 Object Storage endpoint",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for FileLu S5 Object Storage.\nRequired when using FileLu S5.",
@@ -1517,6 +1559,26 @@ func init() {
 				Value: "s3.ru-1.storage.selcloud.ru",
 				Help:  "Saint Petersburg",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Servercore Object Storage.",
+			Provider: "Servercore",
+			Examples: []fs.OptionExample{{
+				Value: "s3.ru-1.storage.selcloud.ru",
+				Help:  "Saint Petersburg",
+			}, {
+				Value: "s3.gis-1.storage.selcloud.ru",
+				Help:  "Moscow",
+			}, {
+				Value: "s3.ru-7.storage.selcloud.ru",
+				Help:  "Moscow",
+			}, {
+				Value: "s3.uz-2.srvstorage.uz",
+				Help:  "Tashkent, Uzbekistan",
+			}, {
+				Value: "s3.kz-1.srvstorage.kz",
+				Help:  "Almaty, Kazakhstan",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for Scaleway Object Storage.",
@@ -1647,7 +1709,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,GCS,Hetzner,HuaweiOBS,IBMCOS,IDrive,Intercolo,IONOS,Liara,Linode,LyveCloud,Magalu,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,StackPath,Storj,Synology,TencentCOS,Zata",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cubbit,GCS,Hetzner,HuaweiOBS,IBMCOS,IDrive,Intercolo,IONOS,Liara,Linode,LyveCloud,Magalu,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,Servercore,StackPath,Storj,Synology,TencentCOS,Zata",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -2196,7 +2258,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,FileLu,HuaweiOBS,IBMCOS,IDrive,Intercolo,IONOS,Leviia,Liara,Linode,Magalu,Mega,Outscale,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,SpectraLogic,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,Cubbit,FlashBlade,FileLu,HuaweiOBS,IBMCOS,IDrive,Intercolo,IONOS,Leviia,Liara,Linode,Magalu,Mega,Outscale,OVHcloud,Petabox,Qiniu,Rabata,RackCorp,Scaleway,Selectel,Servercore,SpectraLogic,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -2211,7 +2273,7 @@ doesn't copy the ACL from the source but rather writes a fresh one.
 If the acl is an empty string then no X-Amz-Acl: header is added and
 the default (private) will be used.
 `,
-			Provider: "!Cloudflare,FlashBlade,Mega,Rabata,Selectel,SpectraLogic,Storj,Synology",
+			Provider: "!Cloudflare,FlashBlade,Mega,Rabata,Selectel,Servercore,SpectraLogic,Storj,Synology",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
 				Help:     "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -3785,6 +3847,9 @@ func setQuirks(opt *Options) {
 	case "Cloudflare":
 		virtualHostStyle = false
 		useMultipartEtag = false // currently multipart Etags are random
+	case "Cubbit":
+		// no quirks
+		useMultipartEtag = false // Cubbit calculates multipart Etags differently from AWS
 	case "ArvanCloud":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -3884,6 +3949,8 @@ func setQuirks(opt *Options) {
 		useAlreadyExists = true
 	case "Selectel":
 		urlEncodeListings = false
+	case "Servercore":
+		urlEncodeListings = false
 	case "SeaweedFS":
 		listObjectsV2 = false // untested
 		virtualHostStyle = false

bin/markdown-lint

@@ -14,4 +14,4 @@ if [ -z "$globs" ]; then
     exit 1
 fi
 
-docker run -v $PWD:/workdir --user $(id -u):$(id -g) davidanson/markdownlint-cli2 $globs
+docker run --rm -v $PWD:/workdir --user $(id -u):$(id -g) davidanson/markdownlint-cli2 $globs

docs/content/_index.md

@@ -117,6 +117,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
 {{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="Cloudinary" home="https://cloudinary.com/" config="/cloudinary/" >}}
+{{< provider name="Cubbit DS3" home="https://cubbit.io/ds3-cloud" config="/s3/#Cubbit" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -192,6 +193,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}}
 {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
 {{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
+{{< provider name="Servercore Object Storage" home="https://servercore.com/services/object-storage/" config="/s3/#servercore" >}}
 {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
 {{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}}
 {{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}}

docs/content/authors.md

@@ -1021,3 +1021,4 @@ put them back in again.` >}}
 - Andrew Ruthven <andrew@etc.gen.nz>
 - spiffytech <git@spiffy.tech>
 - Dulani Woods <Dulani@gmail.com>
+- Marco Ferretti <mferretti93@gmail.com>

docs/content/docs.md

@@ -852,6 +852,32 @@ the binary units, e.g. 1, 2\*\*10, 2\*\*20, 2\*\*30 respectively.
 
 See also [--human-readable](#human-readable).
 
+### --assume-listings-sorted
+
+This flag can be used when the source and destination backends are
+guaranteed to return the items in the same sorted order and in that
+case it will speed up the sync.
+
+Not all backends are guaranteed to return sorted entries (eg local)
+but s3 should, so an s3 to s3 sync could benefit from this flag.
+
+If rclone finds an out of order directory entry then it will cancel
+the sync with the error:
+
+```console
+out of order listing in source (remote:dir)
+```
+
+In this case you should remove the `--assume-listings-sorted` flag.
+
+If you are using `--assume-listings-sorted` then rclone will assume
+`--no-unicode-normalization` and it will compare file names in a case
+sensitive way.
+
+Normally sorting directory entries is not a bottleneck, but it can
+become so with syncs of millions of items in a single directory as the
+sync will not start until the directory listing is complete.
+
 ## Main options
 
 ### --backup-dir string

docs/content/s3.md

@@ -18,6 +18,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
 {{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud" >}}
+{{< provider name="Cubbit DS3" home="https://cubbit.io/ds3-cloud" config="/s3/#Cubbit" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
 {{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}}
@@ -47,6 +48,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}}
 {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
 {{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
+{{< provider name="Servercore Object Storage" home="https://servercore.com/services/object-storage/" config="/s3/#servercore" >}}
 {{< provider name="Spectra Logic" home="https://spectralogic.com/blackpearl-nearline-object-gateway" config="/s3/#spectralogic" >}}
 {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
 {{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
@@ -3234,6 +3236,47 @@ does. If this is causing a problem then upload the files with
 A consequence of this is that `Content-Encoding: gzip` will never
 appear in the metadata on Cloudflare.
 
+### Cubbit DS3 {#Cubbit}
+
+[Cubbit Object Storage](https://www.cubbit.io/ds3-cloud) is a geo-distributed cloud object storage platform.
+
+To connect to Cubbit DS3 you will need an access key and secret key pair. You can follow this [guide](https://docs.cubbit.io/getting-started/quickstart#api-keys) to retrieve these keys. They will be needed when prompted by `rclone config`.
+
+Default region will correspond to `eu-west-1` and the endpoint has to be specified as `s3.cubbit.eu`.
+
+Going through the whole process of creating a new remote by running `rclone config`, each prompt should be answered as shown below:
+
+```
+name> cubbit-ds3 (or any name you like)
+Storage> s3
+provider> Cubbit
+env_auth> false
+access_key_id> YOUR_ACCESS_KEY
+secret_access_key> YOUR_SECRET_KEY
+region> eu-west-1 (or leave empty)
+endpoint> s3.cubbit.eu
+acl>
+```
+
+The resulting configuration file should look like:
+
+```
+[cubbit-ds3]
+type = s3
+provider = Cubbit
+access_key_id = ACCESS_KEY
+secret_access_key = SECRET_KEY
+region = eu-west-1
+endpoint = s3.cubbit.eu
+```
+
+You can then start using Cubbit DS3 with rclone. For example, to create a new bucket and copy files into it, you can run:
+
+```
+rclone mkdir cubbit-ds3:my-bucket
+rclone copy /path/to/files cubbit-ds3:my-bucket
+```
+
 ### DigitalOcean Spaces
 
 [Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean.
@@ -6404,6 +6447,117 @@ region = ru-1
 endpoint = s3.ru-1.storage.selcloud.ru
 ```
 
+### Servercore {#servercore}
+[Servercore Object Storage](https://servercore.io/object-storage/) is an S3
+compatible object storage system that provides scalable and secure storage
+solutions for businesses of all sizes.
+
+rclone config example:
+
+```
+No remotes found, make a new one\?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter name for new remote.
+name> servercore
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including ..., Servercore, ...
+   \ (s3)
+[snip]
+Storage> s3
+
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / Servercore Object Storage
+   \ (Servercore)
+[snip]
+provider> Servercore
+
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> SECRET_ACCESS_KEY
+
+Option region.
+Region where your is data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / St. Petersburg
+   \ (ru-1)
+ 2 / Moscow
+   \ (gis-1)
+ 3 / Moscow
+   \ (ru-7)
+ 4 / Tashkent, Uzbekistan
+   \ (uz-2)
+ 5 / Almaty, Kazakhstan
+   \ (kz-1)
+region> 1
+
+Option endpoint.
+Endpoint for Servercore Object Storage.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Saint Petersburg
+   \ (s3.ru-1.storage.selcloud.ru)
+ 2 / Moscow
+   \ (s3.gis-1.storage.selcloud.ru)
+ 3 / Moscow
+   \ (s3.ru-7.storage.selcloud.ru)
+ 4 / Tashkent, Uzbekistan
+   \ (s3.uz-2.srvstorage.uz)
+ 5 / Almaty, Kazakhstan
+   \ (s3.kz-1.srvstorage.kz)
+endpoint> 1
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: s3
+- provider: Servercore
+- access_key_id: ACCESS_KEY
+- secret_access_key: SECRET_ACCESS_KEY
+- region: ru-1
+- endpoint: s3.ru-1.storage.selcloud.ru
+Keep this "servercore" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
 ### Spectra Logic {#spectralogic}
 
 [Spectra Logic](https://www.spectralogic.com/blackpearl-nearline-object-gateway)

docs/content/sponsor.md

@@ -59,7 +59,6 @@ Thank you very much to our sponsors:
 <!-- markdownlint-capture -->
 <!-- markdownlint-disable line-length no-bare-urls -->
 
-{{< sponsor src="/img/logos/backblaze.svg" width="300" height="200" title="Visit our sponsor Backblaze" link="https://www.backblaze.com/cloud-storage-rclonead?utm_source=rclone&utm_medium=paid&utm_campaign=rclone-website-20250715">}}
 {{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
 {{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
 {{< sponsor src="/img/logos/mega-s4.svg" width="300" height="200" title="MEGA S4: New S3 compatible object storage. High scale. Low cost. Free egress." link="https://mega.io/objectstorage?utm_source=rclone&utm_medium=referral&utm_campaign=rclone-mega-s4&mct=rclonepromo">}}
@@ -67,7 +66,7 @@ Thank you very much to our sponsors:
 {{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
 {{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
 {{< sponsor src="/img/logos/rcloneui.svg" width="300" height="200" title="Visit our sponsor RcloneUI" link="https://github.com/rclone-ui/rclone-ui">}}
-{{< sponsor src="/img/logos/filelu-rclone.svg" width="250" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
+{{< sponsor src="/img/logos/filelu-rclone.svg" width="300" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
 {{< sponsor src="/img/logos/torbox.png" width="200" height="200" title="Visit our sponsor TORBOX" link="https://www.torbox.app/">}}
 
 <!-- markdownlint-restore -->

docs/layouts/chrome/menu.html

@@ -9,15 +9,6 @@
 </div>
 {{end}}
 
-<div class="card">
-  <div class="card-header">
-    Platinum Sponsor
-  </div>
-  <div class="card-body">
-    <a href="https://www.backblaze.com/cloud-storage-rclonead?utm_source=rclone&utm_medium=paid&utm_campaign=rclone-website-20250715" target="_blank" rel="noopener" title="Visit rclone's sponsor Backblaze"><img src="/img/logos/backblaze.svg"></a><br />
-  </div>
-</div>
-
 <div class="card">
   <div class="card-header">
     Gold Sponsor

fs/config.go

@@ -566,6 +566,12 @@ var ConfigOptionsInfo = Options{{
 	Default: "",
 	Help:    "HTTP proxy URL.",
 	Groups:  "Networking",
+}, {
+	Name:     "assume_listings_sorted",
+	Default:  false,
+	Advanced: true,
+	Help:     "If set will not sort listings. If listings aren't sorted the sync may go wrong.",
+	Groups:   "Copy",
 }}
 
 // ConfigInfo is filesystem config options
@@ -680,6 +686,7 @@ type ConfigInfo struct {
 	MaxConnections             int               `config:"max_connections"`
 	NameTransform              []string          `config:"name_transform"`
 	HTTPProxy                  string            `config:"http_proxy"`
+	AssumeListingsSorted       bool              `config:"assume_listings_sorted"`
 }
 
 func init() {

fs/list/sorter.go

@@ -45,6 +45,7 @@ type Sorter struct {
 	keyFn      KeyFn                 // transform an entry into a sort key
 	cutoff     int                   // number of entries above which we start extsort
 	extSort    bool                  // true if we are ext sorting
+	noSort     bool                  // true if we aren't sorting
 	inputChan  chan string           // for sending data to the ext sort
 	outputChan <-chan string         // for receiving data from the ext sort
 	errChan    <-chan error          // for getting errors from the ext sort
@@ -78,6 +79,7 @@ func NewSorter(ctx context.Context, f NewObjecter, callback fs.ListRCallback, ke
 		keyFn:    keyFn,
 		cutoff:   ci.ListCutoff,
 		errs:     errcount.New(),
+		noSort:   ci.AssumeListingsSorted,
 	}, nil
 }
 
@@ -172,6 +174,9 @@ func (ls *Sorter) startExtSort() (err error) {
 //
 // Safe to call from concurrent go routines
 func (ls *Sorter) Add(entries fs.DirEntries) error {
+	if ls.noSort {
+		return ls.callback(entries)
+	}
 	ls.mu.Lock()
 	defer ls.mu.Unlock()
 	if ls.extSort {
@@ -267,6 +272,9 @@ func (lh *listHelper) Flush() error {
 
 // Send the sorted entries to the callback.
 func (ls *Sorter) Send() (err error) {
+	if ls.noSort {
+		return nil
+	}
 	ls.mu.Lock()
 	defer ls.mu.Unlock()
 

fs/list/sorter_test.go

@@ -46,6 +46,46 @@ func TestSorter(t *testing.T) {
 	assert.Equal(t, fs.DirEntries(nil), ls.entries)
 }
 
+func TestSorterAssumeSorted(t *testing.T) {
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.AssumeListingsSorted = true
+
+	gotEntry := 0
+	wantEntries := fs.DirEntries{
+		mockdir.New("c"),
+		mockobject.Object("C"),
+		mockdir.New("b"),
+		mockobject.Object("B"),
+		mockdir.New("a"),
+		mockobject.Object("A"),
+	}
+	callback := func(entries fs.DirEntries) error {
+		for _, entry := range entries {
+			require.Equal(t, wantEntries[gotEntry], entry)
+			gotEntry++
+		}
+		return nil
+	}
+	ls, err := NewSorter(ctx, nil, callback, nil)
+	require.NoError(t, err)
+
+	// Test Add
+	require.NoError(t, ls.Add(wantEntries[0:2]))
+	require.NoError(t, ls.Add(wantEntries[2:6]))
+	assert.Equal(t, 6, gotEntry)
+	assert.Equal(t, fs.DirEntries(nil), ls.entries)
+
+	// Test Send
+	err = ls.Send()
+	require.NoError(t, err)
+	assert.Equal(t, 6, gotEntry)
+
+	// Test Cleanup
+	ls.CleanUp()
+	assert.Equal(t, 6, gotEntry)
+	assert.Equal(t, fs.DirEntries(nil), ls.entries)
+}
+
 func TestSorterIdentity(t *testing.T) {
 	ctx := context.Background()
 	cmpFn := func(a, b fs.DirEntry) int {

fs/march/march.go

@@ -13,6 +13,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/dirtree"
 	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/transform"
@@ -330,7 +331,8 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
 				continue
 			} else if srcName < srcPrevName {
 				// this should never happen since we sort the listings
-				panic("Out of order listing in source")
+				// however the user may be using the --assume-listings-sorted flag
+				return fserrors.FatalError(fmt.Errorf("out of order listing in source (%v)", src.Fs()))
 			}
 		}
 		if dst != nil && dstPrev != nil {
@@ -340,7 +342,8 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
 				continue
 			} else if dstName < dstPrevName {
 				// this should never happen since we sort the listings
-				panic("Out of order listing in destination")
+				// however the user may be using the --assume-listings-sorted flag
+				return fserrors.FatalError(fmt.Errorf("out of order listing in destination (%v)", dst.Fs()))
 			}
 		}
 		switch {