fs/march: fix runtime: program exceeds 10000-thread limit

Before this change when doing a sync with `--no-traverse` and
`--files-from` we could call `NewObject` a total of `--checkers` *
`--checkers` times simultaneously.

With `--checkers 128` this can exceed the 10,000 thread limit and
fails when run on a local to local transfer because `NewObject` calls
`lstat` which is a syscall which needs an OS thread of its own.

This patch uses a weighted semaphore to limit the number of
simultaneous calls to `NewObject` to `--checkers` instead which won't
blow the 10,000 thread limit and is far more sensible use of OS
resources.

Fixes #9073

README.md

@@ -32,8 +32,9 @@ directories to and from different cloud storage providers.
 - Box [:page_facing_up:](https://rclone.org/box/)
 - Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
-- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
 - Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
+- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
+- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
 - DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
 - Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)

docs/content/authors.md

@@ -1060,3 +1060,6 @@ put them back in again. -->
 - jhasse-shade <jacob@shade.inc>
 - vyv03354 <VYV03354@nifty.ne.jp>
 - masrlinu <masrlinu@users.noreply.github.com> <5259918+masrlinu@users.noreply.github.com>
+- vupn0712 <126212736+vupn0712@users.noreply.github.com>
+- darkdragon-001 <darkdragon-001@users.noreply.github.com>
+- sys6101 <csvmen@gmail.com>

docs/content/sponsor.md

@@ -13,7 +13,7 @@ Thank you to our sponsors:
 <!-- markdownlint-capture -->
 <!-- markdownlint-disable line-length no-bare-urls -->
 
-{{< sponsor src="/img/logos/rabata/txt_1_300x114.png" width="300" height="200" title="Visit our sponsor Rabata.io" link="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general">}}
+{{< sponsor src="/img/logos/rabata.svg" width="300" height="200" title="Visit our sponsor Rabata.io" link="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general">}}
 {{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
 {{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
 {{< sponsor src="/img/logos/mega-s4.svg" width="300" height="200" title="MEGA S4: New S3 compatible object storage. High scale. Low cost. Free egress." link="https://mega.io/objectstorage?utm_source=rclone&utm_medium=referral&utm_campaign=rclone-mega-s4&mct=rclonepromo">}}

docs/layouts/chrome/menu.html

@@ -14,20 +14,7 @@
     Platinum Sponsor
   </div>
   <div class="card-body">
-    <a id="platinum" href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img style="width: 100%; height: auto;" src="/img/logos/rabata/txt_1_website.png"></a><br />
+    <a href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img src="/img/logos/rabata.svg"></a><br />
-    <script>
-      const imgs = [
-        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_1_website.png" },
-        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_2_website.png" },
-        { href: "https://rabata.io/grant-application?utm_source=banner&utm_medium=rclone&utm_content=grant1", img: "/img/logos/rabata/100k_website.png" },
-      ];
-      const img = imgs[Math.floor(Math.random() * imgs.length)];
-      document.addEventListener("DOMContentLoaded", () => {
-        const a = document.getElementById("platinum");
-        a.href = img.href;
-        a.querySelector("img").src = img.img;
-      });
-    </script>
   </div>
 </div>
 

fs/march/march.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/transform"
+	"golang.org/x/sync/semaphore"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -41,9 +42,10 @@ type March struct {
 	NoCheckDest            bool            // transfer all objects regardless without checking dst
 	NoUnicodeNormalization bool            // don't normalize unicode characters in filenames
 	// internal state
-	srcListDir listDirFn // function to call to list a directory in the src
+	srcListDir   listDirFn // function to call to list a directory in the src
-	dstListDir listDirFn // function to call to list a directory in the dst
+	dstListDir   listDirFn // function to call to list a directory in the dst
-	transforms []matchTransformFn
+	transforms   []matchTransformFn
+	newObjectSem *semaphore.Weighted // make sure we don't call too many NewObjects simultaneously
 }
 
 // Marcher is called on each match
@@ -78,6 +80,8 @@ func (m *March) init(ctx context.Context) {
 	if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
 		m.transforms = append(m.transforms, strings.ToLower)
 	}
+	// Only allow ci.Checkers simultaneous calls to NewObject
+	m.newObjectSem = semaphore.NewWeighted(int64(ci.Checkers))
 }
 
 // srcOrDstKey turns a directory entry into a sort key using the defined transforms.
@@ -461,7 +465,12 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
 						continue
 					}
 					leaf := path.Base(t.src.Remote())
+					if err := m.newObjectSem.Acquire(m.Ctx, 1); err != nil {
+						t.dstMatch <- nil
+						continue
+					}
 					dst, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))
+					m.newObjectSem.Release(1)
 					if err != nil {
 						dst = nil
 					}