vfs: remove virtual directory entries on backends which can have empty dirs

Before this change we only removed virtual directory entries when they
appeared in the listing.

This works fine except for when virtual directory entries are deleted
outside rclone.

This change deletes directory virtual directory entries for backends
which can have empty directories before reading the directory.

See: https://forum.rclone.org/t/google-drive-rclone-rc-operations-mkdir-fails-on-repeats/17787

backend/s3/s3.go

@@ -1188,9 +1188,6 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		return nil, nil, errors.New("secret_access_key not found")
 	}
 
-	if opt.Region == "" && opt.Endpoint == "" {
-		opt.Endpoint = "https://s3.amazonaws.com/"
-	}
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}

bin/make_manual.py

@@ -6,6 +6,7 @@ conversion into man pages etc.
 
 import os
 import re
+import time
 from datetime import datetime
 
 docpath = "docs/content"
@@ -156,17 +157,19 @@ def read_commands(docpath):
         if command != "rclone.md":
             docs.append(read_command(command))
     return "\n".join(docs)
-    
+
 def main():
     check_docs(docpath)
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
+    build_date = datetime.utcfromtimestamp(
+            int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-""" % datetime.now().strftime("%b %d, %Y"))
+""" % build_date.strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

docs/content/onedrive.md

@@ -127,7 +127,7 @@ requests.
 If you are having problems with them (E.g., seeing a lot of throttling), you can get your own
 Client ID and Key by following the steps below:
 
-1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade, then click `New registration`.
+1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI` Enter `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Copy and keep that secret for later use.
 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.

vfs/dir.go

@@ -42,9 +42,11 @@ type Dir struct {
 type vState byte
 
 const (
-	vOK  vState = iota // Not virtual
-	vAdd               // added file or directory
-	vDel               // removed file or directory
+	vOK      vState = iota // Not virtual
+	vAddFile               // added file
+	vDelFile               // removed file
+	vAddDir                // added directory
+	vDelDir                // removed directory
 )
 
 func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
@@ -295,8 +297,12 @@ func (d *Dir) addObject(node Node) {
 	if d.virtual == nil {
 		d.virtual = make(map[string]vState)
 	}
-	d.virtual[leaf] = vAdd
-	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vAdd, leaf)
+	virtualState := vAddFile
+	if node.IsDir() {
+		virtualState = vAddDir
+	}
+	d.virtual[leaf] = virtualState
+	fs.Debugf(d.path, "Added virtual directory entry %v: %q", virtualState, leaf)
 	d.mu.Unlock()
 }
 
@@ -339,8 +345,8 @@ func (d *Dir) delObject(leaf string) {
 	if d.virtual == nil {
 		d.virtual = make(map[string]vState)
 	}
-	d.virtual[leaf] = vDel
-	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)
+	d.virtual[leaf] = vDelFile
+	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDelFile, leaf)
 	d.mu.Unlock()
 }
 
@@ -392,6 +398,22 @@ func (d *Dir) _readDirFromDirTree(dirTree dirtree.DirTree, when time.Time) error
 // set the last read time - must be called with the lock held
 func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree, when time.Time) error {
 	var err error
+
+	// For backends which can have empty directories remove
+	// virtual directory entries before doing the listing - they
+	// should definitely appear in the listing.
+	if d.f.Features().CanHaveEmptyDirectories {
+		for name, virtualState := range d.virtual {
+			if virtualState == vAddDir {
+				delete(d.virtual, name)
+				if len(d.virtual) == 0 {
+					d.virtual = nil
+				}
+				fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
+			}
+		}
+	}
+
 	// Cache the items by name
 	found := make(map[string]struct{})
 	for _, entry := range entries {
@@ -403,7 +425,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 		found[name] = struct{}{}
 		virtualState := d.virtual[name]
 		switch virtualState {
-		case vAdd:
+		case vAddFile, vAddDir:
 			// item was added to the dir but since it is found in a
 			// listing is no longer virtual
 			delete(d.virtual, name)
@@ -411,7 +433,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 				d.virtual = nil
 			}
 			fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
-		case vDel:
+		case vDelFile, vDelDir:
 			// item is deleted from the dir so skip it
 			continue
 		case vOK:
@@ -453,7 +475,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 	}
 	// delete unused entries
 	for name := range d.items {
-		if _, ok := found[name]; !ok && d.virtual[name] != vAdd {
+		if _, ok := found[name]; !ok && d.virtual[name] != vAddFile && d.virtual[name] != vAddDir {
 			// item was added to the dir but wasn't found in the
 			// listing - remove it unless it was virtually added
 			delete(d.items, name)
@@ -461,7 +483,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 	}
 	// delete unused virtuals
 	for name, virtualState := range d.virtual {
-		if _, ok := found[name]; !ok && virtualState == vDel {
+		if _, ok := found[name]; !ok && (virtualState == vDelFile || virtualState == vDelDir) {
 			// We have a virtual delete but the item wasn't found in
 			// the listing so no longer needs a virtual delete.
 			delete(d.virtual, name)

vfs/vstate_string.go

@@ -9,13 +9,15 @@ func _() {
 	// Re-run the stringer command to generate them again.
 	var x [1]struct{}
 	_ = x[vOK-0]
-	_ = x[vAdd-1]
-	_ = x[vDel-2]
+	_ = x[vAddFile-1]
+	_ = x[vDelFile-2]
+	_ = x[vAddDir-3]
+	_ = x[vDelDir-4]
 }
 
-const _vState_name = "vOKvAddvDel"
+const _vState_name = "vOKvAddFilevDelFilevAddDirvDelDir"
 
-var _vState_index = [...]uint8{0, 3, 7, 11}
+var _vState_index = [...]uint8{0, 3, 11, 19, 26, 33}
 
 func (i vState) String() string {
 	if i >= vState(len(_vState_index)-1) {