fshttp: add --dump curl for dumping HTTP requests as curl commands

README.md

@@ -32,9 +32,8 @@ directories to and from different cloud storage providers.
 - Box [:page_facing_up:](https://rclone.org/box/)
 - Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
-- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
 - Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
-- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
+- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
 - DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
 - Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)

backend/s3/s3.go

@@ -2928,9 +2928,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	req := s3.CopyObjectInput{
 		MetadataDirective: types.MetadataDirectiveCopy,
 	}
-	if srcObj.storageClass != nil {
-		req.StorageClass = types.StorageClass(*srcObj.storageClass)
-	}
+
 	// Build upload options including headers and metadata
 	ci := fs.GetConfig(ctx)
 	uploadOptions := fs.MetadataAsOpenOptions(ctx)
@@ -4503,12 +4501,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 		ACL:    types.ObjectCannedACL(o.fs.opt.ACL),
 		Key:    &bucketPath,
 	}
-	if tierObj, ok := src.(fs.GetTierer); ok {
-		tier := tierObj.GetTier()
-		if tier != "" {
-			ui.req.StorageClass = types.StorageClass(strings.ToUpper(tier))
-		}
-	}
+
 	// Fetch metadata if --metadata is in use
 	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
 	if err != nil {

docs/content/authors.md

@@ -1060,6 +1060,3 @@ put them back in again. -->
 - jhasse-shade <jacob@shade.inc>
 - vyv03354 <VYV03354@nifty.ne.jp>
 - masrlinu <masrlinu@users.noreply.github.com> <5259918+masrlinu@users.noreply.github.com>
-- vupn0712 <126212736+vupn0712@users.noreply.github.com>
-- darkdragon-001 <darkdragon-001@users.noreply.github.com>
-- sys6101 <csvmen@gmail.com>

docs/content/commands/rclone_mount.md

@@ -336,7 +336,7 @@ full new copy of the file.
 When mounting with `--read-only`, attempts to write to files will fail *silently*
 as opposed to with a clear warning as in macFUSE.
 
-## Mounting on Linux
+# Mounting on Linux
 
 On newer versions of Ubuntu, you may encounter the following error when running
 `rclone mount`:

docs/content/docs.md

@@ -3278,6 +3278,10 @@ The available flags are:
 - `mapper` dumps the JSON blobs being sent to the program supplied with
   `--metadata-mapper` and received from it. It can be useful for debugging
   the metadata mapper interface.
+- `curl` dumps the HTTP request as a `curl` command. Can be used with
+  the other HTTP debugging flags (e.g. `requests`, `bodies`). By
+  default the auth will be masked - use with `auth` to have the curl
+  commands with authentication too.
 
 ## Filtering
 

docs/content/sponsor.md

@@ -13,7 +13,7 @@ Thank you to our sponsors:
 <!-- markdownlint-capture -->
 <!-- markdownlint-disable line-length no-bare-urls -->
 
-{{< sponsor src="/img/logos/rabata.svg" width="300" height="200" title="Visit our sponsor Rabata.io" link="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general">}}
+{{< sponsor src="/img/logos/rabata/txt_1_300x114.png" width="300" height="200" title="Visit our sponsor Rabata.io" link="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general">}}
 {{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
 {{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
 {{< sponsor src="/img/logos/mega-s4.svg" width="300" height="200" title="MEGA S4: New S3 compatible object storage. High scale. Low cost. Free egress." link="https://mega.io/objectstorage?utm_source=rclone&utm_medium=referral&utm_campaign=rclone-mega-s4&mct=rclonepromo">}}

docs/layouts/chrome/menu.html

@@ -14,7 +14,20 @@
     Platinum Sponsor
   </div>
   <div class="card-body">
-    <a href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img src="/img/logos/rabata.svg"></a><br />
+    <a id="platinum" href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img style="width: 100%; height: auto;" src="/img/logos/rabata/txt_1_website.png"></a><br />
+    <script>
+      const imgs = [
+        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_1_website.png" },
+        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_2_website.png" },
+        { href: "https://rabata.io/grant-application?utm_source=banner&utm_medium=rclone&utm_content=grant1", img: "/img/logos/rabata/100k_website.png" },
+      ];
+      const img = imgs[Math.floor(Math.random() * imgs.length)];
+      document.addEventListener("DOMContentLoaded", () => {
+        const a = document.getElementById("platinum");
+        a.href = img.href;
+        a.querySelector("img").src = img.img;
+      });
+    </script>
   </div>
 </div>
 

fs/dump.go

@@ -14,6 +14,7 @@ const (
 	DumpGoRoutines
 	DumpOpenFiles
 	DumpMapper
+	DumpCurl
 )
 
 type dumpChoices struct{}
@@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {
 		{uint64(DumpGoRoutines), "goroutines"},
 		{uint64(DumpOpenFiles), "openfiles"},
 		{uint64(DumpMapper), "mapper"},
+		{uint64(DumpCurl), "curl"},
 	}
 }
 

fs/fshttp/http.go

@@ -15,6 +15,8 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -24,6 +26,7 @@ import (
 	"github.com/rclone/rclone/lib/structs"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/net/publicsuffix"
+	"moul.io/http2curl/v2"
 )
 
 const (
@@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte {
 	return buf
 }
 
+// cleanCurl gets rid of Auth headers in a curl command
+func cleanCurl(cmd *http2curl.CurlCommand) {
+	for _, authBuf := range authBufs {
+		auth := "'" + string(authBuf)
+		for i, arg := range *cmd {
+			if strings.HasPrefix(arg, auth) {
+				(*cmd)[i] = auth + "XXXX'"
+			}
+		}
+	}
+}
+
 var expireWindow = 30 * time.Second
 
 func isCertificateExpired(cc *tls.Config) bool {
@@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorReq)
 		logMutex.Unlock()
 	}
+	// Dump curl request
+	if t.dump&(fs.DumpCurl) != 0 {
+		cmd, err := http2curl.GetCurlCommand(req)
+		if err != nil {
+			fs.Debugf(nil, "Failed to create curl command: %v", err)
+		} else {
+			// Patch -X HEAD into --head
+			for i := range len(*cmd) - 1 {
+				if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" {
+					(*cmd)[i] = "--head"
+					*cmd = slices.Delete(*cmd, i+1, i+2)
+					break
+				}
+			}
+			if t.dump&fs.DumpAuth == 0 {
+				cleanCurl(cmd)
+			}
+			fs.Debugf(nil, "HTTP REQUEST: %v", cmd)
+		}
+	}
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response

fs/fshttp/http_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"moul.io/http2curl/v2"
 )
 
 func TestCleanAuth(t *testing.T) {
@@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) {
 	}
 }
 
+func TestCleanCurl(t *testing.T) {
+	for _, test := range []struct {
+		in   []string
+		want []string
+	}{{
+		[]string{""},
+		[]string{""},
+	}, {
+		[]string{"floo"},
+		[]string{"floo"},
+	}, {
+		[]string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'Authorization: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""},
+	}} {
+		in := http2curl.CurlCommand(test.in)
+		cleanCurl(&in)
+		assert.Equal(t, test.want, test.in, test.in)
+	}
+}
+
 var certSerial = int64(0)
 
 // Create a test certificate and key pair that is valid for a specific

fs/march/march.go

@@ -16,7 +16,6 @@ import (
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/transform"
-	"golang.org/x/sync/semaphore"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -42,10 +41,9 @@ type March struct {
 	NoCheckDest            bool            // transfer all objects regardless without checking dst
 	NoUnicodeNormalization bool            // don't normalize unicode characters in filenames
 	// internal state
-	srcListDir   listDirFn // function to call to list a directory in the src
-	dstListDir   listDirFn // function to call to list a directory in the dst
-	transforms   []matchTransformFn
-	newObjectSem *semaphore.Weighted // make sure we don't call too many NewObjects simultaneously
+	srcListDir listDirFn // function to call to list a directory in the src
+	dstListDir listDirFn // function to call to list a directory in the dst
+	transforms []matchTransformFn
 }
 
 // Marcher is called on each match
@@ -80,8 +78,6 @@ func (m *March) init(ctx context.Context) {
 	if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
 		m.transforms = append(m.transforms, strings.ToLower)
 	}
-	// Only allow ci.Checkers simultaneous calls to NewObject
-	m.newObjectSem = semaphore.NewWeighted(int64(ci.Checkers))
 }
 
 // srcOrDstKey turns a directory entry into a sort key using the defined transforms.
@@ -465,12 +461,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
 						continue
 					}
 					leaf := path.Base(t.src.Remote())
-					if err := m.newObjectSem.Acquire(m.Ctx, 1); err != nil {
-						t.dstMatch <- nil
-						continue
-					}
 					dst, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))
-					m.newObjectSem.Release(1)
 					if err != nil {
 						dst = nil
 					}