onedrive: add support for OAuth client credential flow - fixes #6197

This adds support for the client credential flow oauth method which
requires some special handling in onedrive:

- Special scopes are required
- The tenant is required
- The tenant needs to be used in the oauth URLs

This also:

- refactors the oauth config creation so it isn't duplicated
- defaults the drive_id to the previous one in the config
- updates the documentation

Co-authored-by: Nick Craig-Wood <nick@craig-wood.com>

backend/box/box.go

@@ -46,7 +46,6 @@ import (
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
-	"golang.org/x/oauth2"
 )
 
 const (
@@ -65,12 +64,10 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes:       nil,
-		Endpoint: oauth2.Endpoint{
 		AuthURL:      "https://app.box.com/api/oauth2/authorize",
 		TokenURL:     "https://app.box.com/api/oauth2/token",
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/drive/drive.go

@@ -80,9 +80,10 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	driveConfig = &oauth2.Config{
+	driveConfig = &oauthutil.Config{
 		Scopes:       []string{scopePrefix + "drive"},
-		Endpoint:     google.Endpoint,
+		AuthURL:      google.Endpoint.AuthURL,
+		TokenURL:     google.Endpoint.TokenURL,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/dropbox/dropbox.go

@@ -94,7 +94,7 @@ const (
 
 var (
 	// Description of how to auth for this app
-	dropboxConfig = &oauth2.Config{
+	dropboxConfig = &oauthutil.Config{
 		Scopes: []string{
 			"files.metadata.write",
 			"files.content.write",
@@ -109,7 +109,8 @@ var (
 		// 	AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
 		// 	TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
 		// },
-		Endpoint:     dropbox.OAuthEndpoint(""),
+		AuthURL:      dropbox.OAuthEndpoint("").AuthURL,
+		TokenURL:     dropbox.OAuthEndpoint("").TokenURL,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -134,7 +135,7 @@ var (
 )
 
 // Gets an oauth config with the right scopes
-func getOauthConfig(m configmap.Mapper) *oauth2.Config {
+func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
 	// If not impersonating, use standard scopes
 	if impersonate, _ := m.Get("impersonate"); impersonate == "" {
 		return dropboxConfig

backend/googlecloudstorage/googlecloudstorage.go

@@ -60,14 +60,17 @@ const (
 	minSleep                    = 10 * time.Millisecond
 )
 
-// Description of how to auth for this app
-var storageConfig = &oauth2.Config{
+var (
+	// Description of how to auth for this app
+	storageConfig = &oauthutil.Config{
 		Scopes:       []string{storage.DevstorageReadWriteScope},
-	Endpoint:     google.Endpoint,
+		AuthURL:      google.Endpoint.AuthURL,
+		TokenURL:     google.Endpoint.TokenURL,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
-}
+	}
+)
 
 // Register with Fs
 func init() {

backend/googlephotos/googlephotos.go

@@ -33,7 +33,6 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 )
 
@@ -60,13 +59,14 @@ const (
 
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes: []string{
 			"openid",
 			"profile",
 			scopeReadWrite, // this must be at position scopeAccess
 		},
-		Endpoint:     google.Endpoint,
+		AuthURL:      google.Endpoint.AuthURL,
+		TokenURL:     google.Endpoint.TokenURL,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/hidrive/hidrive.go

@@ -31,7 +31,6 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 )
 
 const (
@@ -48,11 +47,9 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app.
-	oauthConfig = &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
+	oauthConfig = &oauthutil.Config{
 		AuthURL:      "https://my.hidrive.com/client/authorize",
 		TokenURL:     "https://my.hidrive.com/oauth2/token",
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.TitleBarRedirectURL,

backend/jottacloud/jottacloud.go

@@ -277,11 +277,9 @@ machines.`)
 		m.Set(configClientID, teliaseCloudClientID)
 		m.Set(configTokenURL, teliaseCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
+			OAuth2Config: &oauthutil.Config{
 				AuthURL:     teliaseCloudAuthURL,
 				TokenURL:    teliaseCloudTokenURL,
-				},
 				ClientID:    teliaseCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -292,11 +290,9 @@ machines.`)
 		m.Set(configClientID, telianoCloudClientID)
 		m.Set(configTokenURL, telianoCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
+			OAuth2Config: &oauthutil.Config{
 				AuthURL:     telianoCloudAuthURL,
 				TokenURL:    telianoCloudTokenURL,
-				},
 				ClientID:    telianoCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -307,11 +303,9 @@ machines.`)
 		m.Set(configClientID, tele2CloudClientID)
 		m.Set(configTokenURL, tele2CloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
+			OAuth2Config: &oauthutil.Config{
 				AuthURL:     tele2CloudAuthURL,
 				TokenURL:    tele2CloudTokenURL,
-				},
 				ClientID:    tele2CloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -322,11 +316,9 @@ machines.`)
 		m.Set(configClientID, onlimeCloudClientID)
 		m.Set(configTokenURL, onlimeCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
-			OAuth2Config: &oauth2.Config{
-				Endpoint: oauth2.Endpoint{
+			OAuth2Config: &oauthutil.Config{
 				AuthURL:     onlimeCloudAuthURL,
 				TokenURL:    onlimeCloudTokenURL,
-				},
 				ClientID:    onlimeCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
@@ -924,19 +916,17 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 	}
 
 	baseClient := fshttp.NewClient(ctx)
-	oauthConfig := &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
+	oauthConfig := &oauthutil.Config{
 		AuthURL:  defaultTokenURL,
 		TokenURL: defaultTokenURL,
-		},
 	}
 	if ver == configVersion {
 		oauthConfig.ClientID = defaultClientID
 		// if custom endpoints are set use them else stick with defaults
 		if tokenURL, ok := m.Get(configTokenURL); ok {
-			oauthConfig.Endpoint.TokenURL = tokenURL
+			oauthConfig.TokenURL = tokenURL
 			// jottacloud is weird. we need to use the tokenURL as authURL
-			oauthConfig.Endpoint.AuthURL = tokenURL
+			oauthConfig.AuthURL = tokenURL
 		}
 	} else if ver == legacyConfigVersion {
 		clientID, ok := m.Get(configClientID)
@@ -950,8 +940,8 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
 		oauthConfig.ClientID = clientID
 		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-		oauthConfig.Endpoint.TokenURL = legacyTokenURL
-		oauthConfig.Endpoint.AuthURL = legacyTokenURL
+		oauthConfig.TokenURL = legacyTokenURL
+		oauthConfig.AuthURL = legacyTokenURL
 
 		// add the request filter to fix token refresh
 		if do, ok := baseClient.Transport.(interface {

backend/mailru/mailru.go

@@ -68,14 +68,12 @@ var (
 )
 
 // Description of how to authorize
-var oauthConfig = &oauth2.Config{
+var oauthConfig = &oauthutil.Config{
 	ClientID:     api.OAuthClientID,
 	ClientSecret: "",
-	Endpoint: oauth2.Endpoint{
 	AuthURL:      api.OAuthURL,
 	TokenURL:     api.OAuthURL,
 	AuthStyle:    oauth2.AuthStyleInParams,
-	},
 }
 
 // Register with Fs
@@ -438,7 +436,9 @@ func (f *Fs) authorize(ctx context.Context, force bool) (err error) {
 	if err != nil || !tokenIsValid(t) {
 		fs.Infof(f, "Valid token not found, authorizing.")
 		ctx := oauthutil.Context(ctx, f.cli)
-		t, err = oauthConfig.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password)
+
+		oauth2Conf := oauthConfig.MakeOauth2Config()
+		t, err = oauth2Conf.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password)
 	}
 	if err == nil && !tokenIsValid(t) {
 		err = errors.New("invalid token")

backend/onedrive/onedrive.go

@@ -40,7 +40,6 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 )
 
 const (
@@ -65,14 +64,21 @@ const (
 
 // Globals
 var (
-	authPath  = "/common/oauth2/v2.0/authorize"
-	tokenPath = "/common/oauth2/v2.0/token"
+
+	// Define the paths used for token operations
+	commonPathPrefix = "/common" // prefix for the paths if tenant isn't known
+	authPath         = "/oauth2/v2.0/authorize"
+	tokenPath        = "/oauth2/v2.0/token"
 
 	scopeAccess             = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "Sites.Read.All", "offline_access"}
 	scopeAccessWithoutSites = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"}
 
-	// Description of how to auth for this app for a business account
-	oauthConfig = &oauth2.Config{
+	// When using client credential OAuth flow, scope of .default is required in order
+	// to use the permissions configured for the application within the tenant
+	scopeAccessClientCred = fs.SpaceSepList{".default"}
+
+	// Base config for how to auth
+	oauthConfig = &oauthutil.Config{
 		Scopes:       scopeAccess,
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
@@ -183,6 +189,14 @@ Choose or manually enter a custom space separated list with all scopes, that rcl
 					Help:  "Read and write access to all resources, without the ability to browse SharePoint sites. \nSame as if disable_site_permission was set to true",
 				},
 			},
+		}, {
+			Name: "tenant",
+			Help: `ID of the service principal's tenant. Also called its directory ID.
+
+Set this if using
+- Client Credential flow
+`,
+			Sensitive: true,
 		}, {
 			Name: "disable_site_permission",
 			Help: `Disable the request for Sites.Read.All permission.
@@ -527,28 +541,54 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 	})
 }
 
-// Config the backend
-func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-	region, graphURL := getRegionURL(m)
+// Make the oauth config for the backend
+func makeOauthConfig(ctx context.Context, opt *Options) (*oauthutil.Config, error) {
+	// Copy the default oauthConfig
+	oauthConfig := *oauthConfig
 
-	if config.State == "" {
-		var accessScopes fs.SpaceSepList
-		accessScopesString, _ := m.Get("access_scopes")
-		err := accessScopes.Set(accessScopesString)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse access_scopes: %w", err)
-		}
-		oauthConfig.Scopes = []string(accessScopes)
-		disableSitePermission, _ := m.Get("disable_site_permission")
-		if disableSitePermission == "true" {
+	// Set the scopes
+	oauthConfig.Scopes = opt.AccessScopes
+	if opt.DisableSitePermission {
 		oauthConfig.Scopes = scopeAccessWithoutSites
 	}
-		oauthConfig.Endpoint = oauth2.Endpoint{
-			AuthURL:  authEndpoint[region] + authPath,
-			TokenURL: authEndpoint[region] + tokenPath,
+
+	// Construct the auth URLs
+	prefix := commonPathPrefix
+	if opt.Tenant != "" {
+		prefix = "/" + opt.Tenant
+	}
+	oauthConfig.TokenURL = authEndpoint[opt.Region] + prefix + tokenPath
+	oauthConfig.AuthURL = authEndpoint[opt.Region] + prefix + authPath
+
+	// Check to see if we are using client credentials flow
+	if opt.ClientCredentials {
+		// Override scope to .default
+		oauthConfig.Scopes = scopeAccessClientCred
+		if opt.Tenant == "" {
+			return nil, fmt.Errorf("tenant parameter must be set when using %s", config.ConfigClientCredentials)
+		}
+	}
+
+	return &oauthConfig, nil
+}
+
+// Config the backend
+func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.ConfigIn) (*fs.ConfigOut, error) {
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	_, graphURL := getRegionURL(m)
+
+	// Check to see if this is the start of the state machine execution
+	if conf.State == "" {
+		conf, err := makeOauthConfig(ctx, opt)
+		if err != nil {
+			return nil, err
 		}
 		return oauthutil.ConfigOut("choose_type", &oauthutil.Options{
-			OAuth2Config: oauthConfig,
+			OAuth2Config: conf,
 		})
 	}
 
@@ -556,9 +596,11 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure OneDrive: %w", err)
 	}
+
+	// Create a REST client, build on the OAuth client created above
 	srv := rest.NewClient(oAuthClient)
 
-	switch config.State {
+	switch conf.State {
 	case "choose_type":
 		return fs.ConfigChooseExclusiveFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{
 			Value: "onedrive",
@@ -584,7 +626,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 		}})
 	case "choose_type_done":
 		// Jump to next state according to config chosen
-		return fs.ConfigGoto(config.Result)
+		return fs.ConfigGoto(conf.Result)
 	case "onedrive":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
 			opts: rest.Opts{
@@ -602,16 +644,22 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 			},
 		})
 	case "driveid":
-		return fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")
+		out, err := fs.ConfigInput("driveid_end", "config_driveid_fixed", "Drive ID")
+		if err != nil {
+			return out, err
+		}
+		// Default the drive_id to the previous version in the config
+		out.Option.Default, _ = m.Get("drive_id")
+		return out, nil
 	case "driveid_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			finalDriveID: config.Result,
+			finalDriveID: conf.Result,
 		})
 	case "siteid":
 		return fs.ConfigInput("siteid_end", "config_siteid", "Site ID")
 	case "siteid_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: config.Result,
+			siteID: conf.Result,
 		})
 	case "url":
 		return fs.ConfigInput("url_end", "config_site_url", `Site URL
@@ -622,7 +670,7 @@ Examples:
 - "https://XXX.sharepoint.com/teams/ID"
 `)
 	case "url_end":
-		siteURL := config.Result
+		siteURL := conf.Result
 		re := regexp.MustCompile(`https://.*\.sharepoint\.com(/.*)`)
 		match := re.FindStringSubmatch(siteURL)
 		if len(match) == 2 {
@@ -637,12 +685,12 @@ Examples:
 		return fs.ConfigInput("path_end", "config_sharepoint_url", `Server-relative URL`)
 	case "path_end":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			relativePath: config.Result,
+			relativePath: conf.Result,
 		})
 	case "search":
 		return fs.ConfigInput("search_end", "config_search_term", `Search term`)
 	case "search_end":
-		searchTerm := config.Result
+		searchTerm := conf.Result
 		opts := rest.Opts{
 			Method:  "GET",
 			RootURL: graphURL,
@@ -664,10 +712,10 @@ Examples:
 		})
 	case "search_sites":
 		return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
-			siteID: config.Result,
+			siteID: conf.Result,
 		})
 	case "driveid_final":
-		finalDriveID := config.Result
+		finalDriveID := conf.Result
 
 		// Test the driveID and get drive type
 		opts := rest.Opts{
@@ -686,12 +734,12 @@ Examples:
 
 		return fs.ConfigConfirm("driveid_final_end", true, "config_drive_ok", fmt.Sprintf("Drive OK?\n\nFound drive %q of type %q\nURL: %s\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL))
 	case "driveid_final_end":
-		if config.Result == "true" {
+		if conf.Result == "true" {
 			return nil, nil
 		}
 		return fs.ConfigGoto("choose_type")
 	}
-	return nil, fmt.Errorf("unknown state %q", config.State)
+	return nil, fmt.Errorf("unknown state %q", conf.State)
 }
 
 // Options defines the configuration for this backend
@@ -702,7 +750,9 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	RootFolderID            string               `config:"root_folder_id"`
 	DisableSitePermission   bool                 `config:"disable_site_permission"`
+	ClientCredentials       bool                 `config:"client_credentials"`
 	AccessScopes            fs.SpaceSepList      `config:"access_scopes"`
+	Tenant                  string               `config:"tenant"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
 	ListChunk               int64                `config:"list_chunk"`
@@ -990,13 +1040,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
-	oauthConfig.Scopes = opt.AccessScopes
-	if opt.DisableSitePermission {
-		oauthConfig.Scopes = scopeAccessWithoutSites
-	}
-	oauthConfig.Endpoint = oauth2.Endpoint{
-		AuthURL:  authEndpoint[opt.Region] + authPath,
-		TokenURL: authEndpoint[opt.Region] + tokenPath,
+
+	oauthConfig, err := makeOauthConfig(ctx, opt)
+	if err != nil {
+		return nil, err
 	}
 
 	client := fshttp.NewClient(ctx)
@@ -2563,8 +2610,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.New("can't upload content to a OneNote file")
 	}
 
+	// Only start the renewer if we have a valid one
+	if o.fs.tokenRenewer != nil {
 		o.fs.tokenRenewer.Start()
 		defer o.fs.tokenRenewer.Stop()
+	}
 
 	size := src.Size()
 

backend/oracleobjectstorage/options.go

@@ -106,9 +106,9 @@ func newOptions() []fs.Option {
 		Sensitive: true,
 	}, {
 		Name:      "compartment",
-		Help:      "Object storage compartment OCID",
+		Help:      "Specify compartment OCID, if you need to list buckets.\n\nList objects works without compartment OCID.",
 		Provider:  "!no_auth",
-		Required:  true,
+		Required:  false,
 		Sensitive: true,
 	}, {
 		Name:     "region",

backend/pcloud/pcloud.go

@@ -48,12 +48,10 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes:  nil,
-		Endpoint: oauth2.Endpoint{
 		AuthURL: "https://my.pcloud.com/oauth2/authorize",
 		// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -61,8 +59,8 @@ var (
 )
 
 // Update the TokenURL with the actual hostname
-func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
-	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
+func updateTokenURL(oauthConfig *oauthutil.Config, hostname string) {
+	oauthConfig.TokenURL = "https://" + hostname + "/oauth2_token"
 }
 
 // Register with Fs
@@ -79,7 +77,7 @@ func init() {
 				fs.Errorf(nil, "Failed to read config: %v", err)
 			}
 			updateTokenURL(oauthConfig, optc.Hostname)
-			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
+			checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error {
 				if auth == nil || auth.Form == nil {
 					return errors.New("form not found in response")
 				}
@@ -399,14 +397,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 	if err != nil {
 		return nil, fmt.Errorf("open file: %w", err)
 	}
+	if _, err := fileClose(ctx, client, f.pacer, openResult.FileDescriptor); err != nil {
+		return nil, fmt.Errorf("close file: %w", err)
+	}
 
 	writer := &writerAt{
 		ctx:    ctx,
-		client: client,
 		fs:     f,
 		size:   size,
 		remote: remote,
-		fd:     openResult.FileDescriptor,
 		fileID: openResult.Fileid,
 	}
 

backend/pcloud/writer_at.go

@@ -18,21 +18,14 @@ import (
 // writerAt implements fs.WriterAtCloser, adding the OpenWrtierAt feature to pcloud.
 type writerAt struct {
 	ctx    context.Context
-	client *rest.Client
 	fs     *Fs
 	size   int64
 	remote string
-	fd     int64
 	fileID int64
 }
 
 // Close implements WriterAt.Close.
 func (c *writerAt) Close() error {
-	// close fd
-	if _, err := c.fileClose(c.ctx); err != nil {
-		return fmt.Errorf("close fd: %w", err)
-	}
-
 	// Avoiding race conditions: Depending on the tcp connection, there might be
 	// caching issues when checking the size immediately after write.
 	// Hence we try avoiding them by checking the resulting size on a different connection.
@@ -72,8 +65,18 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	inSHA1Bytes := sha1.Sum(buffer)
 	inSHA1 := hex.EncodeToString(inSHA1Bytes[:])
 
+	client, err := c.fs.newSingleConnClient(c.ctx)
+	if err != nil {
+		return 0, fmt.Errorf("create client: %w", err)
+	}
+
+	openResult, err := fileOpen(c.ctx, client, c.fs, c.fileID)
+	if err != nil {
+		return 0, fmt.Errorf("open file: %w", err)
+	}
+
 	// get target hash
-	outChecksum, err := c.fileChecksum(c.ctx, offset, int64(contentLength))
+	outChecksum, err := fileChecksum(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, int64(contentLength))
 	if err != nil {
 		return 0, err
 	}
@@ -89,10 +92,15 @@ func (c *writerAt) WriteAt(buffer []byte, offset int64) (n int, err error) {
 	}
 
 	// upload buffer with offset if necessary
-	if _, err := c.filePWrite(c.ctx, offset, buffer); err != nil {
+	if _, err := filePWrite(c.ctx, client, c.fs.pacer, openResult.FileDescriptor, offset, buffer); err != nil {
 		return 0, err
 	}
 
+	// close fd
+	if _, err := fileClose(c.ctx, client, c.fs.pacer, openResult.FileDescriptor); err != nil {
+		return contentLength, fmt.Errorf("close fd: %w", err)
+	}
+
 	return contentLength, nil
 }
 
@@ -125,11 +133,40 @@ func fileOpenNew(ctx context.Context, c *rest.Client, srcFs *Fs, directoryID, fi
 	return result, nil
 }
 
+// Call pcloud file_open using fileid with O_WRITE flags, see [API Doc.]
+// [API Doc]: https://docs.pcloud.com/methods/fileops/file_open.html
+func fileOpen(ctx context.Context, c *rest.Client, srcFs *Fs, fileID int64) (*api.FileOpenResponse, error) {
+	opts := rest.Opts{
+		Method:           "PUT",
+		Path:             "/file_open",
+		Parameters:       url.Values{},
+		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
+		ExtraHeaders: map[string]string{
+			"Connection": "keep-alive",
+		},
+	}
+	opts.Parameters.Set("fileid", strconv.FormatInt(fileID, 10))
+	opts.Parameters.Set("flags", "0x0002") // O_WRITE
+
+	result := &api.FileOpenResponse{}
+	err := srcFs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := c.CallJSON(ctx, &opts, nil, result)
+		err = result.Error.Update(err)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("open new file descriptor: %w", err)
+	}
+	return result, nil
+}
+
 // Call pcloud file_checksum, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_checksum.html
-func (c *writerAt) fileChecksum(
+func fileChecksum(
 	ctx context.Context,
-	offset, count int64,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd, offset, count int64,
 ) (*api.FileChecksumResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
@@ -140,26 +177,29 @@ func (c *writerAt) fileChecksum(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 	opts.Parameters.Set("count", strconv.FormatInt(count, 10))
 
 	result := &api.FileChecksumResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", c.fd, offset, count, err)
+		return nil, fmt.Errorf("checksum of fd %d with offset %d and size %d: %w", fd, offset, count, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_pwrite, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_pwrite.html
-func (c *writerAt) filePWrite(
+func filePWrite(
 	ctx context.Context,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd int64,
 	offset int64,
 	buf []byte,
 ) (*api.FilePWriteResponse, error) {
@@ -176,24 +216,29 @@ func (c *writerAt) filePWrite(
 			"Connection": "keep-alive",
 		},
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 	opts.Parameters.Set("offset", strconv.FormatInt(offset, 10))
 
 	result := &api.FilePWriteResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, c.fd, offset, err)
+		return nil, fmt.Errorf("write %d bytes to fd %d with offset %d: %w", contentLength, fd, offset, err)
 	}
 	return result, nil
 }
 
 // Call pcloud file_close, see [API Doc.]
 // [API Doc]: https://docs.pcloud.com/methods/fileops/file_close.html
-func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error) {
+func fileClose(
+	ctx context.Context,
+	client *rest.Client,
+	pacer *fs.Pacer,
+	fd int64,
+) (*api.FileCloseResponse, error) {
 	opts := rest.Opts{
 		Method:           "PUT",
 		Path:             "/file_close",
@@ -201,11 +246,11 @@ func (c *writerAt) fileClose(ctx context.Context) (*api.FileCloseResponse, error
 		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
 		Close:            true,
 	}
-	opts.Parameters.Set("fd", strconv.FormatInt(c.fd, 10))
+	opts.Parameters.Set("fd", strconv.FormatInt(fd, 10))
 
 	result := &api.FileCloseResponse{}
-	err := c.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := c.client.CallJSON(ctx, &opts, nil, result)
+	err := pacer.CallNoRetry(func() (bool, error) {
+		resp, err := client.CallJSON(ctx, &opts, nil, result)
 		err = result.Error.Update(err)
 		return shouldRetry(ctx, resp, err)
 	})

backend/pikpak/pikpak.go

@@ -82,13 +82,11 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes:      nil,
-		Endpoint: oauth2.Endpoint{
 		AuthURL:     "https://user.mypikpak.com/v1/auth/signin",
 		TokenURL:    "https://user.mypikpak.com/v1/auth/token",
 		AuthStyle:   oauth2.AuthStyleInParams,
-		},
 		ClientID:    clientID,
 		RedirectURL: oauthutil.RedirectURL,
 	}

backend/premiumizeme/premiumizeme.go

@@ -43,7 +43,6 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 )
 
 const (
@@ -59,12 +58,10 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes:       nil,
-		Endpoint: oauth2.Endpoint{
 		AuthURL:      "https://www.premiumize.me/authorize",
 		TokenURL:     "https://www.premiumize.me/token",
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/putio/putio.go

@@ -13,7 +13,6 @@ import (
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
-	"golang.org/x/oauth2"
 )
 
 /*
@@ -41,12 +40,10 @@ const (
 
 var (
 	// Description of how to auth for this app
-	putioConfig = &oauth2.Config{
+	putioConfig = &oauthutil.Config{
 		Scopes:       []string{},
-		Endpoint: oauth2.Endpoint{
 		AuthURL:      "https://api.put.io/v2/oauth2/authenticate",
 		TokenURL:     "https://api.put.io/v2/oauth2/access_token",
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneObscuredClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,

backend/s3/gen_setfrom.go

@@ -102,5 +102,4 @@ import (
 	genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput))
 	genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput))
 	genSetFrom(new(s3.CopyObjectInput), new(s3.PutObjectInput))
-	genSetFrom(new(types.CompletedPart), new(s3.UploadPartOutput))
 }

backend/s3/s3.go

@@ -2724,26 +2724,6 @@ use |-vv| to see the debug level logs.
 `, "|", "`"),
 			Default:  sdkLogMode(0),
 			Advanced: true,
-		}, {
-			Name: "hash",
-			Help: strings.ReplaceAll(`Set to change the hash/checksum in use
-
-This can be set to one of:
-
-- |`+hash.MD5.String()+`|
-- |`+hash.SHA1.String()+`|
-- |`+hash.SHA256.String()+`|
-- |`+hash.CRC32.String()+`|
-
-To choose the checksum algorithm used by S3 to validate your data. Once the data
-is uploaded the checksum algorithm can only be changed by copying the data.
-
-This also sets the Hash that rclone uses.
-
-See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
-`, "|", "`"),
-			Default:  hash.MD5,
-			Advanced: true,
 		},
 		}})
 }
@@ -2898,7 +2878,6 @@ type Options struct {
 	UseUnsignedPayload    fs.Tristate          `config:"use_unsigned_payload"`
 	SDKLogMode            sdkLogMode           `config:"sdk_log_mode"`
 	DirectoryBucket       bool                 `config:"directory_bucket"`
-	Hash                  hash.Type            `config:"hash"`
 }
 
 // Fs represents a remote s3 server
@@ -2920,7 +2899,6 @@ type Fs struct {
 	versioningMu   sync.Mutex
 	versioning     fs.Tristate // if set bucket is using versions
 	warnCompressed sync.Once   // warn once about compressed files
-	checksum       types.ChecksumAlgorithm // for uploading new objects
 }
 
 // Object describes a s3 object
@@ -2944,7 +2922,6 @@ type Object struct {
 	contentDisposition *string // Content-Disposition: header
 	contentEncoding    *string // Content-Encoding: header
 	contentLanguage    *string // Content-Language: header
-	hash               *string // if hash is set, then this is the alternate hash
 }
 
 // safely dereference the pointer, returning a zero T if nil
@@ -3685,18 +3662,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// The normal API doesn't work for creating directory buckets, so don't try
 		f.opt.NoCheckBucket = true
 	}
-	switch opt.Hash {
-	case hash.MD5:
-		f.checksum = ""
-	case hash.CRC32:
-		f.checksum = types.ChecksumAlgorithmCrc32
-	case hash.SHA1:
-		f.checksum = types.ChecksumAlgorithmSha1
-	case hash.SHA256:
-		f.checksum = types.ChecksumAlgorithmSha256
-	default:
-		return nil, fmt.Errorf("%s is not supported for hash/checksum algorithm", opt.Hash)
-	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
 		ReadMimeType:      true,
@@ -4930,7 +4895,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5 | f.opt.Hash)
+	return hash.Set(hash.MD5)
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
@@ -5643,25 +5608,6 @@ func (o *Object) setMD5FromEtag(etag string) {
 // Hash returns the Md5sum of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.MD5 {
-		if t == o.fs.opt.Hash {
-			if o.hash == nil {
-				err := o.readMetaData(ctx)
-				if err != nil {
-					return "", err
-				}
-			}
-			if o.hash == nil {
-				return "", nil
-			}
-			// FIXME check multipart
-			hashBytes, err := base64.StdEncoding.DecodeString(*o.hash)
-			if err != nil {
-				return "", fmt.Errorf("failed to read hash from response %q: %v", *o.hash, err)
-			} else if 2*len(hashBytes) != hash.Width(o.fs.opt.Hash, false) {
-				return "", fmt.Errorf("failed to read hash from response %q: wrong length", *o.hash)
-			}
-			return hex.EncodeToString(hashBytes), nil
-		}
 		return "", hash.ErrUnsupported
 	}
 	// If decompressing, erase the hash
@@ -5706,9 +5652,6 @@ func (f *Fs) headObject(ctx context.Context, req *s3.HeadObjectInput) (resp *s3.
 	if f.opt.SSECustomerKeyMD5 != "" {
 		req.SSECustomerKeyMD5 = &f.opt.SSECustomerKeyMD5
 	}
-	if f.checksum != "" {
-		req.ChecksumMode = types.ChecksumModeEnabled
-	}
 	err = f.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = f.c.HeadObject(ctx, req)
@@ -5798,21 +5741,11 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
 	o.contentDisposition = resp.ContentDisposition
 	o.contentEncoding = resp.ContentEncoding
 	o.contentLanguage = resp.ContentLanguage
-	if o.fs.opt.Hash == hash.CRC32 {
-		o.hash = resp.ChecksumCRC32
-	}
-	if o.fs.opt.Hash == hash.SHA1 {
-		o.hash = resp.ChecksumSHA1
-	}
-	if o.fs.opt.Hash == hash.SHA256 {
-		o.hash = resp.ChecksumSHA256
-	}
 
 	// If decompressing then size and md5sum are unknown
 	if o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" {
 		o.bytes = -1
 		o.md5 = ""
-		o.hash = nil
 	}
 }
 
@@ -5977,9 +5910,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if o.fs.opt.SSECustomerKeyMD5 != "" {
 		req.SSECustomerKeyMD5 = &o.fs.opt.SSECustomerKeyMD5
 	}
-	if o.fs.checksum != "" {
-		req.ChecksumMode = types.ChecksumModeEnabled
-	}
 	// httpReq, err := s3.NewPresignClient(o.fs.c).PresignGetObject(ctx, &req)
 	// if err != nil {
 	// 	return nil, err
@@ -6158,13 +6088,13 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 }
 
 // add a part number and etag to the completed parts
-func (w *s3ChunkWriter) addCompletedPart(partNum *int32, uout *s3.UploadPartOutput) {
+func (w *s3ChunkWriter) addCompletedPart(partNum *int32, eTag *string) {
 	w.completedPartsMu.Lock()
 	defer w.completedPartsMu.Unlock()
-	var part types.CompletedPart
-	setFrom_typesCompletedPart_s3UploadPartOutput(&part, uout)
-	part.PartNumber = partNum
-	w.completedParts = append(w.completedParts, part)
+	w.completedParts = append(w.completedParts, types.CompletedPart{
+		PartNumber: partNum,
+		ETag:       eTag,
+	})
 }
 
 // addMd5 adds a binary md5 to the md5 calculated so far
@@ -6228,7 +6158,6 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		SSECustomerAlgorithm: w.multiPartUploadInput.SSECustomerAlgorithm,
 		SSECustomerKey:       w.multiPartUploadInput.SSECustomerKey,
 		SSECustomerKeyMD5:    w.multiPartUploadInput.SSECustomerKeyMD5,
-		ChecksumAlgorithm:    w.f.checksum,
 	}
 	if w.f.opt.DirectoryBucket {
 		// Directory buckets do not support "Content-Md5" header
@@ -6255,7 +6184,7 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", chunkNumber+1, currentChunkSize, err)
 	}
 
-	w.addCompletedPart(s3PartNumber, uout)
+	w.addCompletedPart(s3PartNumber, uout.ETag)
 
 	fs.Debugf(w.o, "multipart upload wrote chunk %d with %v bytes and etag %v", chunkNumber+1, currentChunkSize, *uout.ETag)
 	return currentChunkSize, err
@@ -6442,7 +6371,6 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 		Bucket: &bucket,
 		ACL:    types.ObjectCannedACL(o.fs.opt.ACL),
 		Key:    &bucketPath,
-		ChecksumAlgorithm: o.fs.checksum,
 	}
 
 	// Fetch metadata if --metadata is in use

backend/s3/setfrom.go

@@ -285,12 +285,3 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Put
 	a.Tagging = b.Tagging
 	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
 }
-
-// setFrom_typesCompletedPart_s3UploadPartOutput copies matching elements from a to b
-func setFrom_typesCompletedPart_s3UploadPartOutput(a *types.CompletedPart, b *s3.UploadPartOutput) {
-	a.ChecksumCRC32 = b.ChecksumCRC32
-	a.ChecksumCRC32C = b.ChecksumCRC32C
-	a.ChecksumSHA1 = b.ChecksumSHA1
-	a.ChecksumSHA256 = b.ChecksumSHA256
-	a.ETag = b.ETag
-}

backend/sharefile/sharefile.go

@@ -97,7 +97,6 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 )
 
 const (
@@ -115,13 +114,11 @@ const (
 )
 
 // Generate a new oauth2 config which we will update when we know the TokenURL
-func newOauthConfig(tokenURL string) *oauth2.Config {
-	return &oauth2.Config{
+func newOauthConfig(tokenURL string) *oauthutil.Config {
+	return &oauthutil.Config{
 		Scopes:       nil,
-		Endpoint: oauth2.Endpoint{
 		AuthURL:      "https://secure.sharefile.com/oauth/authorize",
 		TokenURL:     tokenURL,
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectPublicSecureURL,
@@ -136,7 +133,7 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
 			oauthConfig := newOauthConfig("")
-			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
+			checkAuth := func(oauthConfig *oauthutil.Config, auth *oauthutil.AuthResult) error {
 				if auth == nil || auth.Form == nil {
 					return errors.New("endpoint not found in response")
 				}
@@ -147,7 +144,7 @@ func init() {
 				}
 				endpoint := "https://" + subdomain + "." + apicp
 				m.Set("endpoint", endpoint)
-				oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
+				oauthConfig.TokenURL = endpoint + tokenPath
 				return nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{

backend/yandex/yandex.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/oauth2"
 )
 
 // oAuth
@@ -47,11 +46,9 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
-		Endpoint: oauth2.Endpoint{
+	oauthConfig = &oauthutil.Config{
 		AuthURL:      "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize
 		TokenURL:     "https://oauth.yandex.com/token",     //same as https://oauth.yandex.ru/token
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,

backend/zoho/zoho.go

@@ -47,7 +47,7 @@ const (
 // Globals
 var (
 	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
+	oauthConfig = &oauthutil.Config{
 		Scopes: []string{
 			"aaaserver.profile.read",
 			"WorkDrive.team.READ",
@@ -55,11 +55,10 @@ var (
 			"WorkDrive.files.ALL",
 			"ZohoFiles.files.ALL",
 		},
-		Endpoint: oauth2.Endpoint{
+
 		AuthURL:      "https://accounts.zoho.eu/oauth/v2/auth",
 		TokenURL:     "https://accounts.zoho.eu/oauth/v2/token",
 		AuthStyle:    oauth2.AuthStyleInParams,
-		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
@@ -276,8 +275,8 @@ func setupRegion(m configmap.Mapper) error {
 	downloadURL = fmt.Sprintf("https://download.zoho.%s/v1/workdrive", region)
 	uploadURL = fmt.Sprintf("https://upload.zoho.%s/workdrive-api/v1", region)
 	accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
-	oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
-	oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
+	oauthConfig.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
+	oauthConfig.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
 	return nil
 }
 

cmd/mountlib/mount.go

@@ -373,6 +373,9 @@ func (m *MountPoint) Mount() (mountDaemon *os.Process, err error) {
 
 	m.ErrChan, m.UnmountFn, err = m.MountFn(m.VFS, m.MountPoint, &m.MountOpt)
 	if err != nil {
+		if len(os.Args) > 0 && strings.HasPrefix(os.Args[0], "/snap/") {
+			return nil, fmt.Errorf("mounting is not supported when running from snap")
+		}
 		return nil, fmt.Errorf("failed to mount FUSE fs: %w", err)
 	}
 	m.MountedOn = time.Now()

cmd/serve/sftp/server.go

@@ -143,9 +143,14 @@ func (s *server) serve() (err error) {
 		authKeysFile := env.ShellExpand(s.opt.AuthorizedKeys)
 		authorizedKeysMap, err = loadAuthorizedKeys(authKeysFile)
 		// If user set the flag away from the default then report an error
-		if err != nil && s.opt.AuthorizedKeys != Opt.AuthorizedKeys {
+		if s.opt.AuthorizedKeys != Opt.AuthorizedKeys {
+			if err != nil {
 				return err
 			}
+			if len(authorizedKeysMap) == 0 {
+				return fmt.Errorf("failed to parse authorized keys")
+			}
+		}
 		fs.Logf(nil, "Loaded %d authorized keys from %q", len(authorizedKeysMap), authKeysFile)
 	}
 
@@ -349,12 +354,11 @@ func loadAuthorizedKeys(authorizedKeysPath string) (authorizedKeysMap map[string
 	authorizedKeysMap = make(map[string]struct{})
 	for len(authorizedKeysBytes) > 0 {
 		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse authorized keys: %w", err)
-		}
+		if err == nil {
 			authorizedKeysMap[string(pubKey.Marshal())] = struct{}{}
 			authorizedKeysBytes = bytes.TrimSpace(rest)
 		}
+	}
 	return authorizedKeysMap, nil
 }
 

docs/content/box.md

@@ -64,7 +64,7 @@ If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
-If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=XXXXXXXXXXXXXXXXXXXXXX
 Log in and authorize rclone for access
 Waiting for code...
 Got code

docs/content/docker.md

@@ -537,6 +537,13 @@ sudo curl -H Content-Type:application/json -XPOST -d {} --unix-socket /run/docke
 ```
 though this is rarely needed.
 
+If the plugin fails to work properly, and only as a last resort after you tried diagnosing with the above methods, you can try clearing the state of the plugin. **Note that all existing rclone docker volumes will probably have to be recreated.** This might be needed because a reinstall don't cleanup existing state files to allow for easy restoration, as stated above.
+```
+docker plugin disable rclone # disable the plugin to ensure no interference
+sudo rm /var/lib/docker-plugins/rclone/cache/docker-plugin.state # removing the plugin state
+docker plugin enable rclone # re-enable the plugin afterward
+```
+
 ## Caveats
 
 Finally I'd like to mention a _caveat with updating volume settings_.

docs/content/onedrive.md

@@ -161,6 +161,27 @@ You may try to [verify you account](https://docs.microsoft.com/en-us/azure/activ
 
 Note: If you have a special region, you may need a different host in step 4 and 5. Here are [some hints](https://github.com/rclone/rclone/blob/bc23bf11db1c78c6ebbf8ea538fbebf7058b4176/backend/onedrive/onedrive.go#L86).
 
+### Using OAuth Client Credential flow
+
+OAuth Client Credential flow will allow rclone to use permissions
+directly associated with the Azure AD Enterprise application, rather
+that adopting the context of an Azure AD user account.
+
+This flow can be enabled by following the steps below:
+
+1. Create the Enterprise App registration in the Azure AD portal and obtain a Client ID and Client Secret as described above.
+2. Ensure that the application has the appropriate permissions and they are assigned as *Application Permissions*
+3. Configure the remote, ensuring that *Client ID* and *Client Secret* are entered correctly.
+4. In the *Advanced Config* section, enter `true` for `client_credentials` and in the `tenant` section enter the tenant ID.
+
+When it comes to choosing the type of the connection work with the
+client credentials flow. In particular the "onedrive" option does not
+work. You can use the "sharepoint" option or if that does not find the
+correct drive ID type it in manually with the "driveid" option.
+
+**NOTE** Assigning permissions directly to the application means that
+anyone with the *Client ID* and *Client Secret* can access your
+OneDrive files. Take care to safeguard these credentials.
 
 ### Modification times and hashes
 

fs/config/config.go

@@ -46,6 +46,9 @@ const (
 	// ConfigTokenURL is the config key used to store the token server endpoint
 	ConfigTokenURL = "token_url"
 
+	// ConfigClientCredentials - use OAUTH2 client credentials
+	ConfigClientCredentials = "client_credentials"
+
 	// ConfigEncoding is the config key to change the encoding for a backend
 	ConfigEncoding = "encoding"
 

lib/oauthutil/oauthutil.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -23,6 +24,7 @@ import (
 	"github.com/rclone/rclone/lib/random"
 	"github.com/skratchdot/open-golang/open"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/clientcredentials"
 )
 
 var (
@@ -85,6 +87,49 @@ All done. Please go back to rclone.
 // should work for most uses, but may be overridden.
 var OpenURL = open.Start
 
+// Config - structure that we will use to store the OAuth configuration
+// settings. This is based on the union of the configuration structures for the two
+// OAuth modules that we are using (oauth2 and oauth2.clientcrentials), along with a
+// flag indicating if we are going to use the client credential flow
+type Config struct {
+	ClientID             string
+	ClientSecret         string
+	TokenURL             string
+	AuthURL              string
+	Scopes               []string
+	EndpointParams       url.Values
+	RedirectURL          string
+	ClientCredentialFlow bool
+	AuthStyle            oauth2.AuthStyle
+}
+
+// MakeOauth2Config makes an oauth2.Config from our config
+func (conf *Config) MakeOauth2Config() *oauth2.Config {
+	return &oauth2.Config{
+		ClientID:     conf.ClientID,
+		ClientSecret: conf.ClientSecret,
+		RedirectURL:  RedirectLocalhostURL,
+		Scopes:       conf.Scopes,
+		Endpoint: oauth2.Endpoint{
+			AuthURL:   conf.AuthURL,
+			TokenURL:  conf.TokenURL,
+			AuthStyle: conf.AuthStyle,
+		},
+	}
+}
+
+// MakeClientCredentialsConfig makes a clientcredentials.Config from our config
+func (conf *Config) MakeClientCredentialsConfig() *clientcredentials.Config {
+	return &clientcredentials.Config{
+		ClientID:     conf.ClientID,
+		ClientSecret: conf.ClientSecret,
+		Scopes:       conf.Scopes,
+		TokenURL:     conf.TokenURL,
+		AuthStyle:    conf.AuthStyle,
+		// EndpointParams url.Values
+	}
+}
+
 // SharedOptions are shared between backends the utilize an OAuth flow
 var SharedOptions = []fs.Option{{
 	Name:      config.ConfigClientID,
@@ -107,6 +152,11 @@ var SharedOptions = []fs.Option{{
 	Name:     config.ConfigTokenURL,
 	Help:     "Token server url.\n\nLeave blank to use the provider defaults.",
 	Advanced: true,
+}, {
+	Name:     config.ConfigClientCredentials,
+	Default:  false,
+	Help:     "Use client credentials OAuth flow.\n\nThis will use the OAUTH2 client Credentials Flow as described in RFC 6749.",
+	Advanced: true,
 }}
 
 // oldToken contains an end-user's tokens.
@@ -178,7 +228,7 @@ type TokenSource struct {
 	m           configmap.Mapper
 	tokenSource oauth2.TokenSource
 	token       *oauth2.Token
-	config      *oauth2.Config
+	config      *Config
 	ctx         context.Context
 	expiryTimer *time.Timer // signals whenever the token expires
 }
@@ -264,6 +314,11 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) {
 	)
 	const maxTries = 5
 
+	// If we have a cached valid token, use that
+	if ts.token.Valid() {
+		return ts.token, nil
+	}
+
 	// Try getting the token a few times
 	for i := 1; i <= maxTries; i++ {
 		// Try reading the token from the config file in case it has
@@ -271,7 +326,7 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) {
 		if !ts.token.Valid() {
 			if ts.reReadToken() {
 				changed = true
-			} else if ts.token.RefreshToken == "" {
+			} else if !ts.config.ClientCredentialFlow && ts.token.RefreshToken == "" {
 				return nil, fserrors.FatalError(
 					fmt.Errorf("token expired and there's no refresh token - manually refresh with \"rclone config reconnect %s:\"", ts.name),
 				)
@@ -280,7 +335,11 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) {
 
 		// Make a new token source if required
 		if ts.tokenSource == nil {
-			ts.tokenSource = ts.config.TokenSource(ts.ctx, ts.token)
+			if ts.config.ClientCredentialFlow {
+				ts.tokenSource = ts.config.MakeClientCredentialsConfig().TokenSource(ts.ctx)
+			} else {
+				ts.tokenSource = ts.config.MakeOauth2Config().TokenSource(ts.ctx, ts.token)
+			}
 		}
 
 		token, err = ts.tokenSource.Token()
@@ -297,7 +356,7 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) {
 	if err != nil {
 		return nil, fmt.Errorf("couldn't fetch token: %w", err)
 	}
-	changed = changed || token.AccessToken != ts.token.AccessToken || token.RefreshToken != ts.token.RefreshToken || token.Expiry != ts.token.Expiry
+	changed = changed || ts.token == nil || token.AccessToken != ts.token.AccessToken || token.RefreshToken != ts.token.RefreshToken || token.Expiry != ts.token.Expiry
 	ts.token = token
 	if changed {
 		// Bump on the expiry timer if it is set
@@ -370,12 +429,12 @@ func Context(ctx context.Context, client *http.Client) context.Context {
 	return context.WithValue(ctx, oauth2.HTTPClient, client)
 }
 
-// overrideCredentials sets the ClientID and ClientSecret from the
+// OverrideCredentials sets the ClientID and ClientSecret from the
 // config file if they are not blank.
 // If any value is overridden, true is returned.
 // the origConfig is copied
-func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) {
-	newConfig = new(oauth2.Config)
+func OverrideCredentials(name string, m configmap.Mapper, origConfig *Config) (newConfig *Config, changed bool) {
+	newConfig = new(Config)
 	*newConfig = *origConfig
 	changed = false
 	ClientID, ok := m.Get(config.ConfigClientID)
@@ -393,12 +452,22 @@ func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Con
 	}
 	AuthURL, ok := m.Get(config.ConfigAuthURL)
 	if ok && AuthURL != "" {
-		newConfig.Endpoint.AuthURL = AuthURL
+		newConfig.AuthURL = AuthURL
 		changed = true
 	}
 	TokenURL, ok := m.Get(config.ConfigTokenURL)
 	if ok && TokenURL != "" {
-		newConfig.Endpoint.TokenURL = TokenURL
+		newConfig.TokenURL = TokenURL
+		changed = true
+	}
+	ClientCredentialStr, ok := m.Get(config.ConfigClientCredentials)
+	if ok && ClientCredentialStr != "" {
+		ClientCredential, err := strconv.ParseBool(ClientCredentialStr)
+		if err != nil {
+			fs.Errorf(nil, "Invalid setting for %q: %v", config.ConfigClientCredentials, err)
+		} else {
+			newConfig.ClientCredentialFlow = ClientCredential
+		}
 		changed = true
 	}
 	return newConfig, changed
@@ -408,8 +477,8 @@ func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Con
 // configures a Client with it.  It returns the client and a
 // TokenSource which Invalidate may need to be called on.  It uses the
 // httpClient passed in as the base client.
-func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mapper, config *oauth2.Config, baseClient *http.Client) (*http.Client, *TokenSource, error) {
-	config, _ = overrideCredentials(name, m, config)
+func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mapper, config *Config, baseClient *http.Client) (*http.Client, *TokenSource, error) {
+	config, _ = OverrideCredentials(name, m, config)
 	token, err := GetToken(name, m)
 	if err != nil {
 		return nil, nil, err
@@ -428,12 +497,39 @@ func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mappe
 		ctx:    ctx,
 	}
 	return oauth2.NewClient(ctx, ts), ts, nil
+}
 
+// NewClientCredentialsClient creates a new OAuth module using the
+// ClientCredential flow
+func NewClientCredentialsClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, baseClient *http.Client) (*http.Client, *TokenSource, error) {
+	oauthConfig, _ = OverrideCredentials(name, m, oauthConfig)
+	token, _ := GetToken(name, m)
+	// If the token doesn't exist then we will fetch one in the next step as we don't need a refresh token
+
+	// Set our own http client in the context
+	ctx = Context(ctx, baseClient)
+
+	// Wrap the TokenSource in our TokenSource which saves changed
+	// tokens in the config file
+	ts := &TokenSource{
+		name:   name,
+		m:      m,
+		token:  token,
+		config: oauthConfig,
+		ctx:    ctx,
+	}
+	return oauth2.NewClient(ctx, ts), ts, nil
 }
 
 // NewClient gets a token from the config file and configures a Client
-// with it.  It returns the client and a TokenSource which Invalidate may need to be called on
-func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
+// with it. It returns the client and a TokenSource which Invalidate
+// may need to be called on
+func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config) (*http.Client, *TokenSource, error) {
+	// Check whether we are using the client credentials flow
+	if oauthConfig.ClientCredentialFlow {
+
+		return NewClientCredentialsClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx))
+	}
 	return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx))
 }
 
@@ -460,11 +556,11 @@ func (ar *AuthResult) Error() string {
 }
 
 // CheckAuthFn is called when a good Auth has been received
-type CheckAuthFn func(*oauth2.Config, *AuthResult) error
+type CheckAuthFn func(*Config, *AuthResult) error
 
 // Options for the oauth config
 type Options struct {
-	OAuth2Config *oauth2.Config          // Basic config for oauth2
+	OAuth2Config *Config                 // Basic config for oauth2
 	NoOffline    bool                    // If set then "access_type=offline" parameter is not passed
 	CheckAuth    CheckAuthFn             // When the AuthResult is known the checkAuth function is called if set
 	OAuth2Opts   []oauth2.AuthCodeOption // extra oauth2 options
@@ -532,6 +628,15 @@ func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.Re
 		if in.Result == "false" {
 			return fs.ConfigGoto(newState("*oauth-done"))
 		}
+		opt, err := getOAuth()
+		if err != nil {
+			return nil, err
+		}
+		oauthConfig, _ := OverrideCredentials(name, m, opt.OAuth2Config)
+		if oauthConfig.ClientCredentialFlow {
+			// If using client credential flow, skip straight to getting the token since we don't need a browser
+			return fs.ConfigGoto(newState("*oauth-do"))
+		}
 		return fs.ConfigConfirm(newState("*oauth-islocal"), true, "config_is_local", "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n")
 	case "*oauth-islocal":
 		if in.Result == "true" {
@@ -626,10 +731,16 @@ version recommended):
 		if err != nil {
 			return nil, err
 		}
-		oauthConfig, changed := overrideCredentials(name, m, opt.OAuth2Config)
+		oauthConfig, changed := OverrideCredentials(name, m, opt.OAuth2Config)
 		if changed {
 			fs.Logf(nil, "Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL)
 		}
+		if oauthConfig.ClientCredentialFlow {
+			err = clientCredentialsFlowGetToken(ctx, name, m, oauthConfig, opt)
+			if err != nil {
+				return nil, err
+			}
+		} else {
 			if code == "" {
 				oauthConfig = fixRedirect(oauthConfig)
 				code, err = configSetup(ctx, ri.Name, name, m, oauthConfig, opt)
@@ -641,6 +752,7 @@ version recommended):
 			if err != nil {
 				return nil, err
 			}
+		}
 		return fs.ConfigGoto(newState("*oauth-done"))
 	case "*oauth-done":
 		// Return to the state indicated in the State stack
@@ -656,13 +768,13 @@ func init() {
 }
 
 // Return true if can run without a webserver and just entering a code
-func noWebserverNeeded(oauthConfig *oauth2.Config) bool {
+func noWebserverNeeded(oauthConfig *Config) bool {
 	return oauthConfig.RedirectURL == TitleBarRedirectURL
 }
 
 // get the URL we need to send the user to
-func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (authURL string, state string, err error) {
-	oauthConfig, _ = overrideCredentials(name, m, oauthConfig)
+func getAuthURL(name string, m configmap.Mapper, oauthConfig *Config, opt *Options) (authURL string, state string, err error) {
+	oauthConfig, _ = OverrideCredentials(name, m, oauthConfig)
 
 	// Make random state
 	state, err = random.Password(128)
@@ -670,18 +782,21 @@ func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt
 		return "", "", err
 	}
 
+	// Create the configuration required for the OAuth flow
+	oauth2Conf := oauthConfig.MakeOauth2Config()
+
 	// Generate oauth URL
 	opts := opt.OAuth2Opts
 	if !opt.NoOffline {
 		opts = append(opts, oauth2.AccessTypeOffline)
 	}
-	authURL = oauthConfig.AuthCodeURL(state, opts...)
+	authURL = oauth2Conf.AuthCodeURL(state, opts...)
 	return authURL, state, nil
 }
 
 // If TitleBarRedirect is set but we are doing a real oauth, then
 // override our redirect URL
-func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config {
+func fixRedirect(oauthConfig *Config) *Config {
 	switch oauthConfig.RedirectURL {
 	case TitleBarRedirectURL:
 		// copy the config and set to use the internal webserver
@@ -692,12 +807,33 @@ func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config {
 	return oauthConfig
 }
 
+// configSetup does the initial creation of the token for the client credentials flow
+//
+// If opt is nil it will use the default Options.
+func clientCredentialsFlowGetToken(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, opt *Options) error {
+	if opt == nil {
+		opt = &Options{}
+	}
+	_ = opt // not currently using the Options
+	fs.Debugf(nil, "Getting token for client credentials flow")
+	_, tokenSource, err := NewClientCredentialsClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx))
+	if err != nil {
+		return fmt.Errorf("client credentials flow: failed to make client: %w", err)
+	}
+	// Get the token and save it in the config file
+	_, err = tokenSource.Token()
+	if err != nil {
+		return fmt.Errorf("client credentials flow: failed to get token: %w", err)
+	}
+	return nil
+}
+
 // configSetup does the initial creation of the token
 //
 // If opt is nil it will use the default Options.
 //
 // It will run an internal webserver to receive the results
-func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (string, error) {
+func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *Config, opt *Options) (string, error) {
 	if opt == nil {
 		opt = &Options{}
 	}
@@ -749,9 +885,13 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth
 }
 
 // Exchange the code for a token
-func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config, code string) error {
+func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *Config, code string) error {
 	ctx = Context(ctx, fshttp.NewClient(ctx))
-	token, err := oauthConfig.Exchange(ctx, code)
+
+	// Create the configuration required for the OAuth flow
+	oauth2Conf := oauthConfig.MakeOauth2Config()
+
+	token, err := oauth2Conf.Exchange(ctx, code)
 	if err != nil {
 		return fmt.Errorf("failed to get token: %w", err)
 	}
@@ -813,10 +953,17 @@ func (s *authServer) handleAuth(w http.ResponseWriter, req *http.Request) {
 	// get code, error if empty
 	code := req.Form.Get("code")
 	if code == "" {
-		reply(http.StatusBadRequest, &AuthResult{
+		err := &AuthResult{
 			Name:        "Auth Error",
 			Description: "No code returned by remote server",
-		})
+		}
+		if errorCode := req.Form.Get("error"); errorCode != "" {
+			err.Description += ": " + errorCode
+		}
+		if errorMessage := req.Form.Get("error_description"); errorMessage != "" {
+			err.Description += ": " + errorMessage
+		}
+		reply(http.StatusBadRequest, err)
 		return
 	}
 

vfs/vfs.go

@@ -626,6 +626,10 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 				return nil
 			})
 			vfs.usage.Used = &usedBySizeAlgorithm
+			// if we read a Total size then we should calculate Free from it
+			if vfs.usage.Total != nil {
+				vfs.usage.Free = nil
+			}
 		}
 		vfs.usageTime = time.Now()
 		if err != nil {