diff --git a/docs/content/docs.md b/docs/content/docs.md index f3269ac61..655098557 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -3278,6 +3278,10 @@ The available flags are: - `mapper` dumps the JSON blobs being sent to the program supplied with `--metadata-mapper` and received from it. It can be useful for debugging the metadata mapper interface. +- `curl` dumps the HTTP request as a `curl` command. Can be used with + the other HTTP debugging flags (e.g. `requests`, `bodies`). By + default the auth will be masked - use with `auth` to have the curl + commands with authentication too. ## Filtering diff --git a/fs/dump.go b/fs/dump.go index a461d660a..a2c974419 100644 --- a/fs/dump.go +++ b/fs/dump.go @@ -14,6 +14,7 @@ const ( DumpGoRoutines DumpOpenFiles DumpMapper + DumpCurl ) type dumpChoices struct{} @@ -29,6 +30,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo { {uint64(DumpGoRoutines), "goroutines"}, {uint64(DumpOpenFiles), "openfiles"}, {uint64(DumpMapper), "mapper"}, + {uint64(DumpCurl), "curl"}, } } diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index 65684f3db..626cdcab2 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -15,6 +15,8 @@ import ( "net/http/httputil" "net/url" "os" + "slices" + "strings" "sync" "time" @@ -24,6 +26,7 @@ import ( "github.com/rclone/rclone/lib/structs" "github.com/youmark/pkcs8" "golang.org/x/net/publicsuffix" + "moul.io/http2curl/v2" ) const ( @@ -439,6 +442,18 @@ func cleanAuths(buf []byte) []byte { return buf } +// cleanCurl gets rid of Auth headers in a curl command +func cleanCurl(cmd *http2curl.CurlCommand) { + for _, authBuf := range authBufs { + auth := "'" + string(authBuf) + for i, arg := range *cmd { + if strings.HasPrefix(arg, auth) { + (*cmd)[i] = auth + "XXXX'" + } + } + } +} + var expireWindow = 30 * time.Second func isCertificateExpired(cc *tls.Config) bool { @@ -492,6 +507,26 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error fs.Debugf(nil, "%s", separatorReq) logMutex.Unlock() } + // Dump curl request + if t.dump&(fs.DumpCurl) != 0 { + cmd, err := http2curl.GetCurlCommand(req) + if err != nil { + fs.Debugf(nil, "Failed to create curl command: %v", err) + } else { + // Patch -X HEAD into --head + for i := range len(*cmd) - 1 { + if (*cmd)[i] == "-X" && (*cmd)[i+1] == "'HEAD'" { + (*cmd)[i] = "--head" + *cmd = slices.Delete(*cmd, i+1, i+2) + break + } + } + if t.dump&fs.DumpAuth == 0 { + cleanCurl(cmd) + } + fs.Debugf(nil, "HTTP REQUEST: %v", cmd) + } + } // Do round trip resp, err = t.Transport.RoundTrip(req) // Logf response diff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.go index 18f191e73..084099c77 100644 --- a/fs/fshttp/http_test.go +++ b/fs/fshttp/http_test.go @@ -19,6 +19,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "moul.io/http2curl/v2" ) func TestCleanAuth(t *testing.T) { @@ -61,6 +62,32 @@ func TestCleanAuths(t *testing.T) { } } +func TestCleanCurl(t *testing.T) { + for _, test := range []struct { + in []string + want []string + }{{ + []string{""}, + []string{""}, + }, { + []string{"floo"}, + []string{"floo"}, + }, { + []string{"'Authorization: AAAAAAAAA'", "'Potato: Help'", ""}, + []string{"'Authorization: XXXX'", "'Potato: Help'", ""}, + }, { + []string{"'X-Auth-Token: AAAAAAAAA'", "'Potato: Help'", ""}, + []string{"'X-Auth-Token: XXXX'", "'Potato: Help'", ""}, + }, { + []string{"'X-Auth-Token: AAAAAAAAA'", "'Authorization: AAAAAAAAA'", "'Potato: Help'", ""}, + []string{"'X-Auth-Token: XXXX'", "'Authorization: XXXX'", "'Potato: Help'", ""}, + }} { + in := http2curl.CurlCommand(test.in) + cleanCurl(&in) + assert.Equal(t, test.want, test.in, test.in) + } +} + var certSerial = int64(0) // Create a test certificate and key pair that is valid for a specific