lib/rest: retry errors due to failed JSON decoding - fixes #8383

These errors are almost always caused by an overloaded proxy somewhere
in the chain returning HTML so they are worth retrying.

backend/s3/s3.go

@@ -934,34 +934,67 @@ func init() {
 				Help:  "The default endpoint\nIran",
 			}},
 		}, {
-			// Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint
+			// Linode endpoints: https://techdocs.akamai.com/cloud-computing/docs/object-storage-product-limits#supported-endpoint-types-by-region
 			Name:     "endpoint",
 			Help:     "Endpoint for Linode Object Storage API.",
 			Provider: "Linode",
 			Examples: []fs.OptionExample{{
+				Value: "nl-ams-1.linodeobjects.com",
+				Help:  "Amsterdam (Netherlands), nl-ams-1",
+			}, {
 				Value: "us-southeast-1.linodeobjects.com",
 				Help:  "Atlanta, GA (USA), us-southeast-1",
+			}, {
+				Value: "in-maa-1.linodeobjects.com",
+				Help:  "Chennai (India), in-maa-1",
 			}, {
 				Value: "us-ord-1.linodeobjects.com",
 				Help:  "Chicago, IL (USA), us-ord-1",
 			}, {
 				Value: "eu-central-1.linodeobjects.com",
 				Help:  "Frankfurt (Germany), eu-central-1",
+			}, {
+				Value: "id-cgk-1.linodeobjects.com",
+				Help:  "Jakarta (Indonesia), id-cgk-1",
+			}, {
+				Value: "gb-lon-1.linodeobjects.com",
+				Help:  "London 2 (Great Britain), gb-lon-1",
+			}, {
+				Value: "us-lax-1.linodeobjects.com",
+				Help:  "Los Angeles, CA (USA), us-lax-1",
+			}, {
+				Value: "es-mad-1.linodeobjects.com",
+				Help:  "Madrid (Spain), es-mad-1",
+			}, {
+				Value: "au-mel-1.linodeobjects.com",
+				Help:  "Melbourne (Australia), au-mel-1",
+			}, {
+				Value: "us-mia-1.linodeobjects.com",
+				Help:  "Miami, FL (USA), us-mia-1",
 			}, {
 				Value: "it-mil-1.linodeobjects.com",
 				Help:  "Milan (Italy), it-mil-1",
 			}, {
 				Value: "us-east-1.linodeobjects.com",
 				Help:  "Newark, NJ (USA), us-east-1",
+			}, {
+				Value: "jp-osa-1.linodeobjects.com",
+				Help:  "Osaka (Japan), jp-osa-1",
 			}, {
 				Value: "fr-par-1.linodeobjects.com",
 				Help:  "Paris (France), fr-par-1",
+			}, {
+				Value: "br-gru-1.linodeobjects.com",
+				Help:  "São Paulo (Brazil), br-gru-1",
 			}, {
 				Value: "us-sea-1.linodeobjects.com",
 				Help:  "Seattle, WA (USA), us-sea-1",
 			}, {
 				Value: "ap-south-1.linodeobjects.com",
-				Help:  "Singapore ap-south-1",
+				Help:  "Singapore, ap-south-1",
+			}, {
+				Value: "sg-sin-1.linodeobjects.com",
+				Help:  "Singapore 2, sg-sin-1",
 			}, {
 				Value: "se-sto-1.linodeobjects.com",
 				Help:  "Stockholm (Sweden), se-sto-1",

cmd/purge/purge.go

@@ -22,6 +22,9 @@ include/exclude filters - everything will be removed.  Use the
 delete files. To delete empty directories only, use command
 [rmdir](/commands/rclone_rmdir/) or [rmdirs](/commands/rclone_rmdirs/).
 
+The concurrency of this operation is controlled by the ` + "`--checkers`" + ` global flag. However, some backends will
+implement this command directly, in which case ` + "`--checkers`" + ` will be ignored.
+
 **Important**: Since this can cause data loss, test first with the
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 `,

docs/content/authors.md

@@ -942,3 +942,5 @@ put them back in again.` >}}
   * Zachary Vorhies <zachvorhies@protonmail.com>
   * Alexander Minbaev <minbaev@gmail.com>
   * Joel K Biju <joelkbiju18@gmail.com>
+  * ll3006 <doublel3006@gmail.com>
+  * jbagwell-akamai <113531113+jbagwell-akamai@users.noreply.github.com>

docs/content/s3.md

@@ -4873,27 +4873,49 @@ Option endpoint.
 Endpoint for Linode Object Storage API.
 Choose a number from below, or type in your own value.
 Press Enter to leave empty.
- 1 / Atlanta, GA (USA), us-southeast-1
+ 1 / Amsterdam (Netherlands), nl-ams-1
+   \ (nl-ams-1.linodeobjects.com)
+ 2 / Atlanta, GA (USA), us-southeast-1
    \ (us-southeast-1.linodeobjects.com)
- 2 / Chicago, IL (USA), us-ord-1
+ 3 / Chennai (India), in-maa-1
+   \ (in-maa-1.linodeobjects.com)
+ 4 / Chicago, IL (USA), us-ord-1
    \ (us-ord-1.linodeobjects.com)
- 3 / Frankfurt (Germany), eu-central-1
+ 5 / Frankfurt (Germany), eu-central-1
    \ (eu-central-1.linodeobjects.com)
- 4 / Milan (Italy), it-mil-1
+ 6 / Jakarta (Indonesia), id-cgk-1
+   \ (id-cgk-1.linodeobjects.com)
+ 7 / London 2 (Great Britain), gb-lon-1
+   \ (gb-lon-1.linodeobjects.com)
+ 8 / Los Angeles, CA (USA), us-lax-1
+   \ (us-lax-1.linodeobjects.com)
+ 9 / Madrid (Spain), es-mad-1
+   \ (es-mad-1.linodeobjects.com)
+10 / Melbourne (Australia), au-mel-1
+   \ (au-mel-1.linodeobjects.com)
+11 / Miami, FL (USA), us-mia-1
+   \ (us-mia-1.linodeobjects.com)
+12 / Milan (Italy), it-mil-1
    \ (it-mil-1.linodeobjects.com)
- 5 / Newark, NJ (USA), us-east-1
+13 / Newark, NJ (USA), us-east-1
    \ (us-east-1.linodeobjects.com)
- 6 / Paris (France), fr-par-1
+14 / Osaka (Japan), jp-osa-1
+   \ (jp-osa-1.linodeobjects.com)
+15 / Paris (France), fr-par-1
    \ (fr-par-1.linodeobjects.com)
- 7 / Seattle, WA (USA), us-sea-1
+16 / São Paulo (Brazil), br-gru-1
+   \ (br-gru-1.linodeobjects.com)
+17 / Seattle, WA (USA), us-sea-1
    \ (us-sea-1.linodeobjects.com)
- 8 / Singapore ap-south-1
+18 / Singapore, ap-south-1
    \ (ap-south-1.linodeobjects.com)
- 9 / Stockholm (Sweden), se-sto-1
+19 / Singapore 2, sg-sin-1
+   \ (sg-sin-1.linodeobjects.com)
+20 / Stockholm (Sweden), se-sto-1
    \ (se-sto-1.linodeobjects.com)
-10 / Washington, DC, (USA), us-iad-1
+21 / Washington, DC, (USA), us-iad-1
    \ (us-iad-1.linodeobjects.com)
-endpoint> 3
+endpoint> 5
 
 Option acl.
 Canned ACL used when creating buckets and storing or copying objects.

docs/content/sponsor.md

@@ -57,7 +57,8 @@ off donation.
 Thank you very much to our sponsors:
 
 {{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
-{{< sponsor src="/img/logos/warp.svg" width="300" height="200" title="Visit our sponsor warp.dev" link="https://www.warp.dev/?utm_source=rclone&utm_medium=referral&utm_campaign=rclone_20231103">}}
+{{< sponsor src="/img/logos/warp.svg" width="285" height="200" title="Visit our sponsor warp.dev" link="https://www.warp.dev/?utm_source=rclone&utm_medium=referral&utm_campaign=rclone_20231103">}}
 {{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}}
 {{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
 {{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
+{{< sponsor src="/img/logos/filelu-rclone.svg" width="330" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}

docs/layouts/shortcodes/sponsor.html

@@ -1,3 +1,3 @@
 <a href="{{ .Get "link" }}" target="_blank" >
-<img width="{{ .Get "width" }}" src="{{ .Get "src" }}" title="{{ .Get "title" }}" style="{{ .Get "style" | safeCSS }}">
+<img width="{{ .Get "width" }}" src="{{ .Get "src" }}" title="{{ .Get "title" }}" styleX="{{ .Get "style" | safeCSS }}" style="margin: 2px; padding: 1px; border: 1px solid #ddd; border-radius: 4px;">
 </a>

fs/fshttp/http.go

@@ -17,7 +17,6 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/lib/structs"
-	"golang.org/x/net/http2"
 	"golang.org/x/net/publicsuffix"
 )
 
@@ -121,29 +120,6 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
 
 	if ci.DisableHTTP2 {
 		t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
-	} else {
-		// Enable HTTP/2
-		err := http2.ConfigureTransport(t)
-		if err != nil {
-			fs.Errorf(nil, "Failed to configure HTTP2: %v", err)
-		} else {
-			// Create a new HTTP/2 transport - use HTTP/1 defaults where appropriate
-			t2 := &http2.Transport{
-				DisableCompression: t.DisableCompression,
-				IdleConnTimeout:    t.IdleConnTimeout,
-				TLSClientConfig:    t.TLSClientConfig,
-				MaxReadFrameSize:   8 * 1024 * 1024, // 16k (default) to 16M.
-				//ReadIdleTimeout:            10 * time.Second, // Check idle connections every 10s
-				//WriteByteTimeout:           5 * time.Second,  // Custom setting
-				//AllowHTTP:                  false,            // Ensure HTTPS-only HTTP/2
-				//StrictMaxConcurrentStreams: true,             // Example setting
-			}
-
-			// Override HTTP/2 handling while keeping existing HTTP/1 support
-			t.TLSNextProto[http2.NextProtoTLS] = func(authority string, c *tls.Conn) http.RoundTripper {
-				return t2
-			}
-		}
 	}
 
 	// customize the transport if required

lib/rest/rest.go

@@ -17,6 +17,7 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -189,7 +190,13 @@ func checkDrainAndClose(r io.ReadCloser, err *error) {
 func DecodeJSON(resp *http.Response, result interface{}) (err error) {
 	defer checkDrainAndClose(resp.Body, &err)
 	decoder := json.NewDecoder(resp.Body)
-	return decoder.Decode(result)
+	err = decoder.Decode(result)
+	if err != nil {
+		// Retry this as it is likely some overloaded server sending HTML instead of JSON
+		contentType := resp.Header.Get("Content-Type")
+		err = fserrors.RetryError(fmt.Errorf("failed to decode JSON from Content-Type: %q: %v", contentType, err))
+	}
+	return err
 }
 
 // DecodeXML decodes resp.Body into result

vfs/vfscache/cache.go

@@ -227,7 +227,10 @@ func (c *Cache) createItemDir(name string) (string, error) {
 
 // getBackend gets a backend for a cache root dir
 func getBackend(ctx context.Context, parentPath string, name string, relativeDirPath string) (fs.Fs, error) {
-	path := fmt.Sprintf(":local,encoding='%v':%s/%s/%s", encoder.OS, parentPath, name, relativeDirPath)
+	// Make sure we turn off the global links flag as it overrides the backend specific one
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = false
+	path := fmt.Sprintf(":local,encoding='%v',links=false:%s/%s/%s", encoder.OS, parentPath, name, relativeDirPath)
 	return fscache.Get(ctx, path)
 }