lib/oauthutil: allow the browser opening function to be overridden

This patch ensures that we pass the context from the CreateBackend
call all the way to the creation of the oauth web server.

This means that when the command has finished the webserver will
definitely be shut down, and if the user abandons it (eg via an rc
call timing out or being cancelled) then it will be shut down too.

backend/onedrive/onedrive.go

@@ -827,7 +827,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 				retry = true
 				fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
 			}
-		case 429: // Too Many Requests.
+		case 429, 503: // Too Many Requests, Server Too Busy
 			// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
 			if values := resp.Header["Retry-After"]; len(values) == 1 && values[0] != "" {
 				retryAfter, parseErr := strconv.Atoi(values[0])

backend/pcloud/pcloud.go

@@ -10,7 +10,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	stdhash "hash"
 	"io"
 	"net/http"
 	"net/url"
@@ -59,8 +58,6 @@ var (
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
 	}
-	// PcloudHashType is the hash.Type for Pcloud
-	PcloudHashType hash.Type
 )
 
 // Update the TokenURL with the actual hostname
@@ -68,45 +65,8 @@ func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
 	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
 }
 
-type pcloudHash struct{}
-
-// newHash returns a new hash.Hash computing the quickXorHash checksum.
-func newHash() stdhash.Hash {
-	return &pcloudHash{}
-}
-
-// Write adds more data to the running hash.
-// It never returns an error.
-func (h *pcloudHash) Write(p []byte) (n int, err error) {
-	return len(p), nil
-}
-
-// Sum appends the current hash to b and returns the resulting slice.
-// It does not change the underlying hash state.
-func (h *pcloudHash) Sum(b []byte) []byte {
-	return []byte{}
-}
-
-// Reset resets the Hash to its initial state.
-func (h *pcloudHash) Reset() {
-}
-
-// Size returns the number of bytes Sum will return.
-func (h *pcloudHash) Size() int {
-	return 8
-}
-
-// BlockSize returns the hash's underlying block size.
-// The Write method must be able to accept any amount
-// of data, but it may operate more efficiently if all writes
-// are a multiple of the block size.
-func (h *pcloudHash) BlockSize() int {
-	return 1024
-}
-
 // Register with Fs
 func init() {
-	PcloudHashType = hash.RegisterHash("pcloud", "PcloudHash", 8, newHash)
 	updateTokenURL(oauthConfig, defaultHostname)
 	fs.Register(&fs.RegInfo{
 		Name:        "pcloud",
@@ -229,7 +189,6 @@ type Object struct {
 	sha1        string    // SHA1 if known
 	sha256      string    // SHA256 if known
 	link        *api.GetFileLinkResult
-	pcloudHash  uint64 // pcloud proprietary hash
 }
 
 // ------------------------------------------------------------
@@ -1051,18 +1010,15 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 }
 
 // Hashes returns the supported hash sets.
-func (f *Fs) Hashes() (hashes hash.Set) {
+func (f *Fs) Hashes() hash.Set {
 	// EU region supports SHA1 and SHA256 (but rclone doesn't
 	// support SHA256 yet).
 	//
 	// https://forum.rclone.org/t/pcloud-to-local-no-hashes-in-common/19440
 	if f.opt.Hostname == "eapi.pcloud.com" {
-		hashes = hash.Set(hash.SHA1 | hash.SHA256)
+		return hash.Set(hash.SHA1 | hash.SHA256)
-	} else {
-		hashes = hash.Set(hash.MD5 | hash.SHA1)
 	}
-	hashes = hashes.Add(PcloudHashType)
+	return hash.Set(hash.MD5 | hash.SHA1)
-	return hashes
 }
 
 // ------------------------------------------------------------
@@ -1117,11 +1073,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 		pHash = &o.sha1
 	case hash.SHA256:
 		pHash = &o.sha256
-	case PcloudHashType:
-		if o.pcloudHash == 0 {
-			return "", nil
-		}
-		return fmt.Sprintf("%016x", o.pcloudHash), nil
 	default:
 		return "", hash.ErrUnsupported
 	}
@@ -1153,7 +1104,6 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	o.size = info.Size
 	o.modTime = info.ModTime()
 	o.id = info.ID
-	o.pcloudHash = info.Hash
 	return nil
 }
 

backend/s3/s3.go

@@ -3470,6 +3470,10 @@ func setQuirks(opt *Options) {
 			opt.ChunkSize = 64 * fs.Mebi
 		}
 		useAlreadyExists = false // returns BucketAlreadyExists
+		// Storj doesn't support multi-part server side copy:
+		// https://github.com/storj/roadmap/issues/40
+		// So make cutoff very large which it does support
+		opt.CopyCutoff = math.MaxInt64
 	case "Synology":
 		useMultipartEtag = false
 		useAlreadyExists = false // untested

lib/oauthutil/oauthutil.go

@@ -80,6 +80,11 @@ All done. Please go back to rclone.
 `
 )
 
+// OpenURL is used when rclone wants to open a browser window
+// for user authentication. It defaults to something which
+// should work for most uses, but may be overridden.
+var OpenURL = open.Start
+
 // SharedOptions are shared between backends the utilize an OAuth flow
 var SharedOptions = []fs.Option{{
 	Name:      config.ConfigClientID,
@@ -706,7 +711,7 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth
 
 	// Prepare webserver
 	server := newAuthServer(opt, bindAddress, state, authURL)
-	err = server.Init()
+	err = server.Init(ctx)
 	if err != nil {
 		return "", fmt.Errorf("failed to start auth webserver: %w", err)
 	}
@@ -716,8 +721,12 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth
 
 	if !authorizeNoAutoBrowser {
 		// Open the URL for the user to visit
-		_ = open.Start(authURL)
+		err := OpenURL(authURL)
+		if err != nil {
+			fs.Errorf(nil, "Failed to open browser automatically (%v) - please go to the following link: %s\n", err, authURL)
+		} else {
 			fs.Logf(nil, "If your browser doesn't open automatically go to the following link: %s\n", authURL)
+		}
 	} else {
 		fs.Logf(nil, "Please go to the following link: %s\n", authURL)
 	}
@@ -758,6 +767,7 @@ type authServer struct {
 	authURL     string
 	server      *http.Server
 	result      chan *AuthResult
+	quit        chan struct{}
 }
 
 // newAuthServer makes the webserver for collecting auth
@@ -768,6 +778,7 @@ func newAuthServer(opt *Options, bindAddress, state, authURL string) *authServer
 		bindAddress: bindAddress,
 		authURL:     authURL, // http://host/auth redirects to here
 		result:      make(chan *AuthResult, 1),
+		quit:        make(chan struct{}),
 	}
 }
 
@@ -830,15 +841,32 @@ func (s *authServer) handleAuth(w http.ResponseWriter, req *http.Request) {
 }
 
 // Init gets the internal web server ready to receive config details
-func (s *authServer) Init() error {
+//
+// The web server will listen until ctx is cancelled or the Stop()
+// method is called
+func (s *authServer) Init(ctx context.Context) error {
 	fs.Debugf(nil, "Starting auth server on %s", s.bindAddress)
 	mux := http.NewServeMux()
 	s.server = &http.Server{
 		Addr:        s.bindAddress,
 		Handler:     mux,
+		BaseContext: func(net.Listener) context.Context { return ctx },
 	}
 	s.server.SetKeepAlivesEnabled(false)
 
+	// Error the server if the context is cancelled
+	go func() {
+		select {
+		case <-ctx.Done():
+			s.result <- &AuthResult{
+				Name:        "Cancelled",
+				Description: ctx.Err().Error(),
+				Err:         ctx.Err(),
+			}
+		case <-s.quit:
+		}
+	}()
+
 	mux.HandleFunc("/auth", func(w http.ResponseWriter, req *http.Request) {
 		state := req.FormValue("state")
 		if state != s.state {
@@ -852,7 +880,8 @@ func (s *authServer) Init() error {
 	mux.HandleFunc("/", s.handleAuth)
 
 	var err error
-	s.listener, err = net.Listen("tcp", s.bindAddress)
+	var lc net.ListenConfig
+	s.listener, err = lc.Listen(ctx, "tcp", s.bindAddress)
 	if err != nil {
 		return err
 	}
@@ -869,6 +898,7 @@ func (s *authServer) Serve() {
 func (s *authServer) Stop() {
 	fs.Debugf(nil, "Closing auth server")
 	close(s.result)
+	close(s.quit)
 	_ = s.listener.Close()
 
 	// close the server