fshttp: add --dump curl for dumping HTTP requests as curl commands

docs/content/docs.md

@@ -3278,6 +3278,10 @@ The available flags are:
 - `mapper` dumps the JSON blobs being sent to the program supplied with
   `--metadata-mapper` and received from it. It can be useful for debugging
   the metadata mapper interface.
+- `curl` dumps the HTTP request as a `curl` command. Can be used with
+  the other HTTP debugging flags (e.g. `requests`, `bodies`). By
+  default the auth will be masked - use with `auth` to have the curl
+  commands with authentication too.
 
 ## Filtering
 

fs/dump.go

@@ -14,6 +14,7 @@ const (
 	DumpGoRoutines
 	DumpOpenFiles
 	DumpMapper
+	DumpCurl
 )
 
 type dumpChoices struct{}
@@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {
 		{uint64(DumpGoRoutines), "goroutines"},
 		{uint64(DumpOpenFiles), "openfiles"},
 		{uint64(DumpMapper), "mapper"},
+		{uint64(DumpCurl), "curl"},
 	}
 }
 

fs/fshttp/http.go

@@ -15,6 +15,8 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -24,6 +26,7 @@ import (
 	"github.com/rclone/rclone/lib/structs"
 	"github.com/youmark/pkcs8"
 	"golang.org/x/net/publicsuffix"
+	"moul.io/http2curl/v2"
 )
 
 const (
@@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte {
 	return buf
 }
 
+// cleanCurl gets rid of Auth headers in a curl command
+func cleanCurl(cmd *http2curl.CurlCommand) {
+	for _, authBuf := range authBufs {
+		auth := "'" + string(authBuf)
+		for i, arg := range *cmd {
+			if strings.HasPrefix(arg, auth) {
+				(*cmd)[i] = auth + "XXXX'"
+			}
+		}
+	}
+}
+
 var expireWindow = 30 * time.Second
 
 func isCertificateExpired(cc *tls.Config) bool {
@@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorReq)
 		logMutex.Unlock()
 	}
+	// Dump curl request
+	if t.dump&(fs.DumpCurl) != 0 {
+		cmd, err := http2curl.GetCurlCommand(req)
+		if err != nil {
+			fs.Debugf(nil, "Failed to create curl command: %v", err)
+		} else {
+			// Patch -X HEAD into --head
+			for i := range len(*cmd) - 1 {
+				if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" {
+					(*cmd)[i] = "--head"
+					*cmd = slices.Delete(*cmd, i+1, i+2)
+					break
+				}
+			}
+			if t.dump&fs.DumpAuth == 0 {
+				cleanCurl(cmd)
+			}
+			fs.Debugf(nil, "HTTP REQUEST: %v", cmd)
+		}
+	}
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response

fs/fshttp/http_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"moul.io/http2curl/v2"
 )
 
 func TestCleanAuth(t *testing.T) {
@@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) {
 	}
 }
 
+func TestCleanCurl(t *testing.T) {
+	for _, test := range []struct {
+		in   []string
+		want []string
+	}{{
+		[]string{""},
+		[]string{""},
+	}, {
+		[]string{"floo"},
+		[]string{"floo"},
+	}, {
+		[]string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'Authorization: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""},
+	}, {
+		[]string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""},
+		[]string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""},
+	}} {
+		in := http2curl.CurlCommand(test.in)
+		cleanCurl(&in)
+		assert.Equal(t, test.want, test.in, test.in)
+	}
+}
+
 var certSerial = int64(0)
 
 // Create a test certificate and key pair that is valid for a specific