fstests: add FsRootCollapse test - #3164

backend/jottacloud/jottacloud.go

@@ -41,7 +41,7 @@ const (
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Archive"
+	defaultMountpoint           = "Sync" // nolint
 	rootURL                     = "https://www.jottacloud.com/jfs/"
 	apiURL                      = "https://api.jottacloud.com/files/v1/"
 	baseURL                     = "https://www.jottacloud.com/"
@@ -53,8 +53,6 @@ const (
 	configUsername              = "user"
 	configClientID              = "client_id"
 	configClientSecret          = "client_secret"
-	configDevice                = "device"
-	configMountpoint            = "mountpoint"
 	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 )
 
@@ -71,7 +69,6 @@ var (
 
 // Register with Fs
 func init() {
-	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
 		Description: "JottaCloud",
@@ -138,7 +135,7 @@ func init() {
 			if !ok {
 				log.Fatalf("No username defined")
 			}
-			password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
+			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
 
 			// prepare out token request with username and password
 			values := url.Values{}
@@ -160,7 +157,7 @@ func init() {
 				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
 				if resp != nil {
 					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
+						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
 						fmt.Printf("Enter verification code> ")
 						authCode := config.ReadLine()
 						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
@@ -183,49 +180,23 @@ func init() {
 			// finally save them in the config
 			err = oauthutil.PutToken(name, m, &token, true)
 			if err != nil {
-				log.Fatalf("Error while saving token: %s", err)
-			}
-
-			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
-			if config.Confirm() {
-				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
-				if err != nil {
-					log.Fatalf("Failed to load oAuthClient: %s", err)
-				}
-
-				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
-
-				acc, err := getAccountInfo(srv, username)
-				if err != nil {
-					log.Fatalf("Error getting devices: %s", err)
-				}
-				fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
-				var deviceNames []string
-				for i := range acc.Devices {
-					deviceNames = append(deviceNames, acc.Devices[i].Name)
-				}
-				result := config.Choose("Devices", deviceNames, nil, false)
-				m.Set(configDevice, result)
-
-				dev, err := getDeviceInfo(srv, path.Join(username, result))
-				if err != nil {
-					log.Fatalf("Error getting Mountpoint: %s", err)
-				}
-				if len(dev.MountPoints) == 0 {
-					log.Fatalf("No Mountpoints found for this device.")
-				}
-				fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
-				var mountpointNames []string
-				for i := range dev.MountPoints {
-					mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
-				}
-				result = config.Choose("Mountpoints", mountpointNames, nil, false)
-				m.Set(configMountpoint, result)
+				log.Fatalf("Error while setting token: %s", err)
 			}
 		},
 		Options: []fs.Option{{
 			Name: configUsername,
 			Help: "User Name:",
+		}, {
+			Name:     "mountpoint",
+			Help:     "The mountpoint to use.",
+			Required: true,
+			Examples: []fs.OptionExample{{
+				Value: "Sync",
+				Help:  "Will be synced by the official client.",
+			}, {
+				Value: "Archive",
+				Help:  "Archive",
+			}},
 		}, {
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -253,7 +224,6 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
-	Device             string        `config:"device"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
@@ -361,31 +331,18 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
 	return &result, nil
 }
 
-// getAccountInfo queries general information about the account.
-// Takes rest.Client and username as parameter to be easily usable
-// during config
-func getAccountInfo(srv *rest.Client, username string) (info *api.AccountInfo, err error) {
+// getAccountInfo retrieves account information
+func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
-		Path:   urlPathEscape(username),
+		Path:   urlPathEscape(f.user),
 	}
 
-	_, err = srv.CallXML(&opts, nil, &info)
-	if err != nil {
-		return nil, err
-	}
-
-	return info, nil
-}
-
-// getDeviceInfo queries Information about a jottacloud device
-func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) {
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   urlPathEscape(path),
-	}
-
-	_, err = srv.CallXML(&opts, nil, &info)
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallXML(&opts, nil, &info)
+		return shouldRetry(resp, err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -394,18 +351,12 @@ func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err er
 }
 
 // setEndpointUrl reads the account id and generates the API endpoint URL
-func (f *Fs) setEndpointURL() (err error) {
-	info, err := getAccountInfo(f.srv, f.user)
+func (f *Fs) setEndpointURL(mountpoint string) (err error) {
+	info, err := f.getAccountInfo()
 	if err != nil {
 		return errors.Wrap(err, "failed to get endpoint url")
 	}
-	if f.opt.Device == "" {
-		f.opt.Device = defaultDevice
-	}
-	if f.opt.Mountpoint == "" {
-		f.opt.Mountpoint = defaultMountpoint
-	}
-	f.endpointURL = urlPathEscape(path.Join(info.Username, f.opt.Device, f.opt.Mountpoint))
+	f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
 	return nil
 }
 
@@ -492,6 +443,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	oauthConfig.ClientID = clientID
 	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
+	// add jottacloud to the long list of sites that don't follow the oauth spec correctly
+	oauth2.RegisterBrokenAuthHeaderProvider("https://www.jottacloud.com/")
+
 	// the oauth client for the api servers needs
 	// a filter to fix the grant_type issues (see above)
 	baseClient := fshttp.NewClient(fs.Config)
@@ -530,7 +484,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return err
 	})
 
-	err = f.setEndpointURL()
+	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
 	}
@@ -788,9 +742,6 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if f.opt.Device != "Jotta" {
-		return nil, errors.New("upload not supported for devices other than Jotta")
-	}
 	o := f.createObject(src.Remote(), src.ModTime(), src.Size())
 	return o, o.Update(in, src, options...)
 }
@@ -1054,7 +1005,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
-	info, err := getAccountInfo(f.srv, f.user)
+	info, err := f.getAccountInfo()
 	if err != nil {
 		return nil, err
 	}

backend/onedrive/onedrive.go

@@ -385,9 +385,19 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var ok bool
-		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
-			dirCacheFoundRoot = true
+		var dirCacheRootIDExists bool
+		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		if f.root == "" {
+			// if f.root == "", it means f.root is the absolute root of the drive
+			// and its ID should have been found in NewFs
+			dirCacheFoundRoot = dirCacheRootIDExists
+		} else if _, err := f.dirCache.RootParentID(); err == nil {
+			// if root is in a folder, it must have a parent folder, and
+			// if dirCache has found root in NewFs, the parent folder's ID
+			// should be present.
+			// This RootParentID() check is a fix for #3164 which describes
+			// a possible case where the root is not found.
+			dirCacheFoundRoot = dirCacheRootIDExists
 		}
 	}
 

docs/content/jottacloud.md

@@ -25,20 +25,26 @@ n) New remote
 s) Set configuration password
 q) Quit config
 n/s/q> n
-name> jotta
+name> remote
 Type of storage to configure.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
 [snip]
-14 / JottaCloud
+13 / JottaCloud
    \ "jottacloud"
 [snip]
 Storage> jottacloud
-** See help for jottacloud backend at: https://rclone.org/jottacloud/ **
-
-User Name:
+User Name
 Enter a string value. Press Enter for the default ("").
-user> user@email.tld
+user> user
+The mountpoint to use.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Will be synced by the official client.
+   \ "Sync"
+ 2 / Archive
+   \ "Archive"
+mountpoint> Archive
 Edit advanced config? (y/n)
 y) Yes
 n) No
@@ -52,35 +58,16 @@ Rclone has it's own Jottacloud API KEY which works fine as long as one only uses
 y) Yes
 n) No
 y/n> y
-Your Jottacloud password is only required during setup and will not be stored.
+Your Jottacloud password is only required during config and will not be stored.
 password:
-
-Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?
-
-y) Yes
-n) No
-y/n> y
-Please select the device to use. Normally this will be Jotta
-Choose a number from below, or type in an existing value
- 1 > DESKTOP-3H31129
- 2 > test1
- 3 > Jotta
-Devices> 3
-Please select the mountpoint to user. Normally this will be Archive
-Choose a number from below, or type in an existing value
- 1 > Archive
- 2 > Shared
- 3 > Sync
-Mountpoints> 1
 --------------------
-[jotta]
+[remote]
 type = jottacloud
-user = 0xC4KE@gmail.com
+user = olihey
+mountpoint = Archive
 client_id = .....
 client_secret = ........
 token = {........}
-device = Jotta
-mountpoint = Archive
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -101,11 +88,6 @@ To copy a local directory to an Jottacloud directory called backup
 
     rclone copy /home/source remote:backup
 
-### Devices and Mountpoints ###
-
-The official Jottacloud client registers a device for each computer you install it on and then creates a mountpoint for each folder you select for Backup.
-The web interface uses a special device called Jotta for the Archive, Sync and Shared mountpoints. In most cases you'll want to use the Jotta/Archive device/mounpoint however if you want to access files uploaded by the official rclone provides the option to select other devices and mountpoints during config.
-
 ### --fast-list ###
 
 This remote supports `--fast-list` which allows you to use fewer
@@ -167,6 +149,20 @@ User Name:
 - Type:        string
 - Default:     ""
 
+#### --jottacloud-mountpoint
+
+The mountpoint to use.
+
+- Config:      mountpoint
+- Env Var:     RCLONE_JOTTACLOUD_MOUNTPOINT
+- Type:        string
+- Default:     ""
+- Examples:
+    - "Sync"
+        - Will be synced by the official client.
+    - "Archive"
+        - Archive
+
 ### Advanced Options
 
 Here are the advanced options specific to jottacloud (JottaCloud).

fstest/fstests/fstests.go

@@ -1583,6 +1583,27 @@ func Run(t *testing.T, opt *Opt) {
 
 		})
 
+		// TestFsRootCollapse tests if the root of an fs "collapses" to the
+		// absolute root. It creates a new fs of the same backend type with its
+		// root set to a *non-existent* folder, and attempts to read the info of
+		// an object in that folder, whose name is taken from a directory that
+		// exists in the absolute root.
+		// This test is added after
+		// https://github.com/ncw/rclone/issues/3164.
+		t.Run("FsRootCollapse", func(t *testing.T) {
+			deepRemoteName := subRemoteName + "/deeper/nonexisting/directory"
+			deepRemote, err := fs.NewFs(deepRemoteName)
+			require.NoError(t, err)
+
+			colonIndex := strings.IndexRune(deepRemoteName, ':')
+			firstSlashIndex := strings.IndexRune(deepRemoteName, '/')
+			firstDir := deepRemoteName[colonIndex+1 : firstSlashIndex]
+			_, err = deepRemote.NewObject(firstDir)
+			require.Equal(t, fs.ErrorObjectNotFound, err)
+			// If err is not fs.ErrorObjectNotFound, it means the backend is
+			// somehow confused about root and absolute root.
+		})
+
 		// Purge the folder
 		err = operations.Purge(remote, "")
 		require.NoError(t, err)