docs: note where a provider has an S3 compatible alternative

Backblaze has updated its b2_authorize_account API endpoint, newly created
application keys are now "multi-bucket" keys, capable of being limited to
multiple buckets. These keys can only be used with the v4 endpoint, not v1 which
returns an HTTP 400.

This commit switches authorization to the v4 endpoint, and allowing such keys to
work with any of the allowed buckets.

With multi-bucket keys, missing restricted buckets can be non-fatal.

Supports listing root with multi-bucket API keys

RELEASE.md

@@ -21,6 +21,7 @@ This file describes how to make the various kinds of releases
 - make doc
 - git status - to check for new man pages - git add them
 - git commit -a -v -m "Version v1.XX.0"
+- make check
 - make retag
 - git push origin # without --follow-tags so it doesn't push the tag if it fails
 - git push --follow-tags origin

VERSION

@@ -1 +1 @@
-v1.72.0
+v1.73.0

backend/b2/api/types.go

@@ -133,23 +133,32 @@ type File struct {
 	Info            map[string]string `json:"fileInfo"`        // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.
 }
 
-// AuthorizeAccountResponse is as returned from the b2_authorize_account call
-type AuthorizeAccountResponse struct {
+// StorageAPI is as returned from the b2_authorize_account call
+type StorageAPI struct {
 	AbsoluteMinimumPartSize int      `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
-	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
-		BucketID     string   `json:"bucketId"`     // When present, access is restricted to one bucket.
-		BucketName   string   `json:"bucketName"`   // When present, name of bucket - may be empty
-		Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
+		Buckets []struct { // When present, access is restricted to one or more buckets.
+			ID   string `json:"id"`   // ID of bucket
+			Name string `json:"name"` // When present, name of bucket - may be empty
+		} `json:"buckets"`
+		Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has for every bucket.
 		NamePrefix   any      `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`
 	APIURL              string `json:"apiUrl"`              // The base URL to use for all API calls except for uploading and downloading files.
-	AuthorizationToken  string `json:"authorizationToken"`  // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
 	DownloadURL         string `json:"downloadUrl"`         // The base URL to use for downloading files.
 	MinimumPartSize     int    `json:"minimumPartSize"`     // DEPRECATED: This field will always have the same value as recommendedPartSize. Use recommendedPartSize instead.
 	RecommendedPartSize int    `json:"recommendedPartSize"` // The recommended size for each part of a large file. We recommend using this part size for optimal upload performance.
 }
 
+// AuthorizeAccountResponse is as returned from the b2_authorize_account call
+type AuthorizeAccountResponse struct {
+	AccountID          string   `json:"accountId"`          // The identifier for the account.
+	AuthorizationToken string   `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
+	APIs               struct { // Supported APIs for this account / key. These are API-dependent JSON objects.
+		Storage StorageAPI `json:"storageApi"`
+	} `json:"apiInfo"`
+}
+
 // ListBucketsRequest is parameters for b2_list_buckets call
 type ListBucketsRequest struct {
 	AccountID   string   `json:"accountId"`             // The identifier for the account.

backend/b2/b2.go

@@ -607,17 +607,29 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("failed to authorize account: %w", err)
 	}
-	// If this is a key limited to a single bucket, it must exist already
-	if f.rootBucket != "" && f.info.Allowed.BucketID != "" {
-		allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)
-		if allowedBucket == "" {
-			return nil, errors.New("bucket that application key is restricted to no longer exists")
+	// If this is a key limited to one or more buckets, one of them must exist
+	// and be ours.
+	if f.rootBucket != "" && len(f.info.APIs.Storage.Allowed.Buckets) != 0 {
+		buckets := f.info.APIs.Storage.Allowed.Buckets
+		var rootFound = false
+		var rootID string
+		for _, b := range buckets {
+			allowedBucket := f.opt.Enc.ToStandardName(b.Name)
+			if allowedBucket == "" {
+				fs.Debugf(f, "bucket %q that application key is restricted to no longer exists", b.ID)
+				continue
+			}
+
+			if allowedBucket == f.rootBucket {
+				rootFound = true
+				rootID = b.ID
+			}
 		}
-		if allowedBucket != f.rootBucket {
-			return nil, fmt.Errorf("you must use bucket %q with this application key", allowedBucket)
+		if !rootFound {
+			return nil, fmt.Errorf("you must use bucket(s) %q with this application key", buckets)
 		}
 		f.cache.MarkOK(f.rootBucket)
-		f.setBucketID(f.rootBucket, f.info.Allowed.BucketID)
+		f.setBucketID(f.rootBucket, rootID)
 	}
 	if f.rootBucket != "" && f.rootDirectory != "" {
 		// Check to see if the (bucket,directory) is actually an existing file
@@ -643,7 +655,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 	defer f.authMu.Unlock()
 	opts := rest.Opts{
 		Method:       "GET",
-		Path:         "/b2api/v1/b2_authorize_account",
+		Path:         "/b2api/v4/b2_authorize_account",
 		RootURL:      f.opt.Endpoint,
 		UserName:     f.opt.Account,
 		Password:     f.opt.Key,
@@ -656,13 +668,13 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to authenticate: %w", err)
 	}
-	f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
+	f.srv.SetRoot(f.info.APIs.Storage.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
 	return nil
 }
 
 // hasPermission returns if the current AuthorizationToken has the selected permission
 func (f *Fs) hasPermission(permission string) bool {
-	return slices.Contains(f.info.Allowed.Capabilities, permission)
+	return slices.Contains(f.info.APIs.Storage.Allowed.Capabilities, permission)
 }
 
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
@@ -1067,44 +1079,68 @@ type listBucketFn func(*api.Bucket) error
 
 // listBucketsToFn lists the buckets to the function supplied
 func (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {
-	var account = api.ListBucketsRequest{
-		AccountID: f.info.AccountID,
-		BucketID:  f.info.Allowed.BucketID,
-	}
-	if bucketName != "" && account.BucketID == "" {
-		account.BucketName = f.opt.Enc.FromStandardName(bucketName)
+	responses := make([]api.ListBucketsResponse, len(f.info.APIs.Storage.Allowed.Buckets))[:0]
+
+	for i := range f.info.APIs.Storage.Allowed.Buckets {
+		b := &f.info.APIs.Storage.Allowed.Buckets[i]
+		// Empty names indicate a bucket that no longer exists, this is non-fatal
+		// for multi-bucket API keys.
+		if b.Name == "" {
+			continue
+		}
+		// When requesting a specific bucket skip over non-matching names
+		if bucketName != "" && b.Name != bucketName {
+			continue
+		}
+
+		var account = api.ListBucketsRequest{
+			AccountID: f.info.AccountID,
+			BucketID:  b.ID,
+		}
+		if bucketName != "" && account.BucketID == "" {
+			account.BucketName = f.opt.Enc.FromStandardName(bucketName)
+		}
+
+		var response api.ListBucketsResponse
+		opts := rest.Opts{
+			Method: "POST",
+			Path:   "/b2_list_buckets",
+		}
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
+			return f.shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return err
+		}
+		responses = append(responses, response)
 	}
 
-	var response api.ListBucketsResponse
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_list_buckets",
-	}
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return err
-	}
 	f.bucketIDMutex.Lock()
 	f.bucketTypeMutex.Lock()
 	f._bucketID = make(map[string]string, 1)
 	f._bucketType = make(map[string]string, 1)
-	for i := range response.Buckets {
-		bucket := &response.Buckets[i]
-		bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
-		f.cache.MarkOK(bucket.Name)
-		f._bucketID[bucket.Name] = bucket.ID
-		f._bucketType[bucket.Name] = bucket.Type
+
+	for ri := range responses {
+		response := &responses[ri]
+		for i := range response.Buckets {
+			bucket := &response.Buckets[i]
+			bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
+			f.cache.MarkOK(bucket.Name)
+			f._bucketID[bucket.Name] = bucket.ID
+			f._bucketType[bucket.Name] = bucket.Type
+		}
 	}
 	f.bucketTypeMutex.Unlock()
 	f.bucketIDMutex.Unlock()
-	for i := range response.Buckets {
-		bucket := &response.Buckets[i]
-		err = fn(bucket)
-		if err != nil {
-			return err
+	for ri := range responses {
+		response := &responses[ri]
+		for i := range response.Buckets {
+			bucket := &response.Buckets[i]
+			err := fn(bucket)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -1606,7 +1642,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	bucket, bucketPath := f.split(remote)
 	var RootURL string
 	if f.opt.DownloadURL == "" {
-		RootURL = f.info.DownloadURL
+		RootURL = f.info.APIs.Storage.DownloadURL
 	} else {
 		RootURL = f.opt.DownloadURL
 	}
@@ -1957,7 +1993,7 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 	// Use downloadUrl from backblaze if downloadUrl is not set
 	// otherwise use the custom downloadUrl
 	if o.fs.opt.DownloadURL == "" {
-		opts.RootURL = o.fs.info.DownloadURL
+		opts.RootURL = o.fs.info.APIs.Storage.DownloadURL
 	} else {
 		opts.RootURL = o.fs.opt.DownloadURL
 	}

docs/content/authors.md

@@ -1048,3 +1048,5 @@ put them back in again. -->
 - jijamik <30904953+jijamik@users.noreply.github.com>
 - Dominik Sander <git@dsander.de>
 - Nikolay Kiryanov <nikolay@kiryanov.ru>
+- Diana <5275194+DianaNites@users.noreply.github.com>
+- Duncan Smart <duncan.smart@gmail.com>

docs/content/b2.md

@@ -283,7 +283,7 @@ It is useful to know how many requests are sent to the server in different scena
 All copy commands send the following 4 requests:
 
 ```text
-/b2api/v1/b2_authorize_account
+/b2api/v4/b2_authorize_account
 /b2api/v1/b2_create_bucket
 /b2api/v1/b2_list_buckets
 /b2api/v1/b2_list_file_names

docs/content/changelog.md

@@ -26,7 +26,7 @@ description: "Rclone Changelog"
   - [rclone test speed](/commands/rclone_test_speed/): Add command to test a specified remotes speed (dougal)
 - New Features
   - backends: many backends have has a paged listing (`ListP`) interface added
-      - this enables progress when listing large directories and reduced memory usage
+    - this enables progress when listing large directories and reduced memory usage
   - build
     - Bump golang.org/x/crypto from 0.43.0 to 0.45.0 to fix CVE-2025-58181 (dependabot[bot])
     - Modernize code and tests (Nick Craig-Wood, russcoss, juejinyuxitu, reddaisyy, dulanting, Oleksandr Redko)

docs/content/drive.md

@@ -265,7 +265,7 @@ account key" button.
   `https://www.googleapis.com/auth/drive`
   to grant read/write access to Google Drive specifically.
   You can also use `https://www.googleapis.com/auth/drive.readonly` for read
-  only access.
+  only access with `--drive-scope=drive.readonly`.
 - Click "Authorise"
 
 ##### 3. Configure rclone, assuming a new install

docs/content/filelu.md

@@ -14,6 +14,9 @@ managing files in the cloud easy. Its cross-platform file backup
 services let you upload and back up files from any internet-connected
 device.
 
+**Note** FileLu now has a fully featured S3 backend [FileLu S5](/s3#filelu-s5),
+an industry standard S3 compatible object store.
+
 ## Configuration
 
 Here is an example of how to make a remote called `filelu`. First, run:

docs/content/sponsor.md

@@ -19,8 +19,9 @@ Thank you to our sponsors:
 {{< sponsor src="/img/logos/mega-s4.svg" width="300" height="200" title="MEGA S4: New S3 compatible object storage. High scale. Low cost. Free egress." link="https://mega.io/objectstorage?utm_source=rclone&utm_medium=referral&utm_campaign=rclone-mega-s4&mct=rclonepromo">}}
 {{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}}
 {{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
-{{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
+{{< sponsor src="/img/logos/rcloneview-banner.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
 {{< sponsor src="/img/logos/rcloneui.svg" width="300" height="200" title="Visit our sponsor RcloneUI" link="https://github.com/rclone-ui/rclone-ui">}}
+{{< sponsor src="/img/logos/shade.svg" width="300" height="200" title="Visit our sponsor Shade" link="https://shade.inc">}}
 {{< sponsor src="/img/logos/filelu-rclone.svg" width="300" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
 {{< sponsor src="/img/logos/torbox.png" width="200" height="200" title="Visit our sponsor TORBOX" link="https://www.torbox.app/">}}
 {{< sponsor src="/img/logos/spectra-logic.svg" width="300" height="200" title="Visit our sponsor Spectra Logic" link="https://spectralogic.com/">}}

docs/layouts/chrome/menu.html

@@ -14,13 +14,12 @@
     Platinum Sponsor
   </div>
   <div class="card-body">
-    <a id="platinum" href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img style="width: 100%; height: auto;" src="/img/logos/rabata/txt_1_300x114.png"></a><br />
+    <a id="platinum" href="https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general" target="_blank" rel="noopener" title="Visit rclone's sponsor Rabata.io"><img style="width: 100%; height: auto;" src="/img/logos/rabata/txt_1_website.png"></a><br />
     <script>
       const imgs = [
-        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_1_300x114.png" },
-        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_2_300x114.png" },
-        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_3_300x114.png" },
-        { href: "https://rabata.io/grant-application?utm_source=banner&utm_medium=rclone&utm_content=grant1", img: "/img/logos/rabata/100k_300x114.png" },
+        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_1_website.png" },
+        { href: "https://rabata.io/?utm_source=banner&utm_medium=rclone&utm_content=general", img: "/img/logos/rabata/txt_2_website.png" },
+        { href: "https://rabata.io/grant-application?utm_source=banner&utm_medium=rclone&utm_content=grant1", img: "/img/logos/rabata/100k_website.png" },
       ];
       const img = imgs[Math.floor(Math.random() * imgs.length)];
       document.addEventListener("DOMContentLoaded", () => {
@@ -67,6 +66,14 @@
     <a href="https://github.com/rclone-ui/rclone-ui" target="_blank" rel="noopener" title="Visit rclone's sponsor rclone UI"><img src="/img/logos/rcloneui.svg"></a><br />
   </div>
 </div>
+<div class="card">
+  <div class="card-header">
+    Silver Sponsor
+  </div>
+  <div class="card-body">
+    <a href="https://shade.inc/" target="_blank" rel="noopener" title="Visit rclone's sponsor Shade"><img style="max-width: 100%; height: auto;" src="/img/logos/shade.svg"></a><br />
+  </div>
+</div>
 {{end}}
 
 <div class="card">

docs/layouts/chrome/navbar.html

@@ -55,7 +55,7 @@
           <a class="dropdown-item" href="/fichier/"><i class="fa fa-archive fa-fw"></i> 1Fichier</a>
           <a class="dropdown-item" href="/netstorage/"><i class="fas fa-database fa-fw"></i> Akamai NetStorage</a>
           <a class="dropdown-item" href="/alias/"><i class="fa fa-link fa-fw"></i> Alias</a>
-          <a class="dropdown-item" href="/s3/"><i class="fab fa-amazon fa-fw"></i> Amazon S3</a>
+          <a class="dropdown-item" href="/s3/"><i class="fab fa-amazon fa-fw"></i> Amazon S3 Storage Providers</a>
           <a class="dropdown-item" href="/archive/"><i class="fa fa-archive fa-fw"></i> Archive</a>
           <a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a>
           <a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a>
@@ -69,6 +69,7 @@
           <a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
           <a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
           <a class="dropdown-item" href="/filelu/"><i class="fa fa-folder fa-fw"></i> FileLu Cloud Storage</a>
+          <a class="dropdown-item" href="/s3/#filelu-s5"><i class="fa fa-folder fa-fw"></i> FileLu S5 (S3-Compatible)</a>
           <a class="dropdown-item" href="/filescom/"><i class="fa fa-brands fa-files-pinwheel fa-fw"></i> Files.com</a>
           <a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>
           <a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a>
@@ -87,7 +88,7 @@
           <a class="dropdown-item" href="/linkbox/"><i class="fa fa-infinity fa-fw"></i> Linkbox</a>
           <a class="dropdown-item" href="/mailru/"><i class="fa fa-at fa-fw"></i> Mail.ru Cloud</a>
           <a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a>
-          <a class="dropdown-item" href="/s3/#mega"><i class="fa fa-archive fa-fw"></i> Mega S4</a>
+          <a class="dropdown-item" href="/s3/#mega"><i class="fa fa-archive fa-fw"></i> Mega S4 (S3-Compatible)</a>
           <a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a>
           <a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a>
           <a class="dropdown-item" href="/azurefiles/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Files Storage</a>

docs/layouts/partials/version.html

@@ -1 +1 @@
-v1.72.0
+v1.73.0

fs/versiontag.go

@@ -1,4 +1,4 @@
 package fs
 
 // VersionTag of rclone
-var VersionTag = "v1.72.0"
+var VersionTag = "v1.73.0"