vfs: rework directory virtuals to calculate them in advance

This had the advantage that they will never be stale.

vfs/dir.go

@@ -25,6 +25,7 @@ type Dir struct {
 	vfs          *VFS        // read only
 	inode        uint64      // read only: inode number
 	f            fs.Fs       // read only
+	cleanupTimer *time.Timer // read only: timer to call cacheCleanup
 
 	mu      sync.RWMutex // protects the following
 	parent  *Dir         // parent, nil for root
@@ -37,6 +38,8 @@ type Dir struct {
 
 	modTimeMu sync.Mutex // protects the following
 	modTime   time.Time
+
+	_childVirtuals atomic.Int32 // non zero if any children have virtual directory entries
 }
 
 //go:generate stringer -type=vState
@@ -52,7 +55,7 @@ const (
 )
 
 func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
-	return &Dir{
+	d := &Dir{
 		vfs:     vfs,
 		f:       f,
 		parent:  parent,
@@ -62,6 +65,25 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
 		inode:   newInode(),
 		items:   make(map[string]Node),
 	}
+	d.cleanupTimer = time.AfterFunc(vfs.Opt.DirCacheTime*2, d.cacheCleanup)
+	return d
+}
+
+func (d *Dir) cacheCleanup() {
+	defer func() {
+		// We should never panic here
+		_ = recover()
+	}()
+
+	when := time.Now()
+
+	d.mu.Lock()
+	_, stale := d._age(when)
+	d.mu.Unlock()
+
+	if stale {
+		d.ForgetAll()
+	}
 }
 
 // String converts it to printable
@@ -174,38 +196,81 @@ func (d *Dir) Node() Node {
 	return d
 }
 
+// hasVirtuals returns whether the directory has virtual entries
+func (d *Dir) hasVirtuals() bool {
+	return d._childVirtuals.Load() != 0
+}
+
+// getVirtuals returns the number of virtual entries in this and children
+func (d *Dir) getVirtuals() int32 {
+	return d._childVirtuals.Load()
+}
+
+// addVirtuals increments or decrements the number of virtual
+// directories by the amount given in this and all the parent
+// directories.
+func (d *Dir) addVirtuals(inc int32) {
+	for {
+		d._childVirtuals.Add(inc)
+		d.mu.RLock()
+		parent := d.parent
+		d.mu.RUnlock()
+		if parent == nil {
+			break
+		}
+		d = parent
+	}
+}
+
+// _addVirtuals increments or decrements the number of virtual
+// directories by the amount given in this and all the parent
+// directories.
+//
+// The dir lock must be held to call this
+func (d *Dir) _addVirtuals(inc int32) {
+	d._childVirtuals.Add(inc)
+	if d.parent == nil {
+		return
+	}
+	d.parent.addVirtuals(inc)
+}
+
 // ForgetAll forgets directory entries for this directory and any children.
 //
 // It does not invalidate or clear the cache of the parent directory.
 //
-// It returns true if the directory or any of its children had virtual entries
+// Directories or parents of directories with virtual entries won't be
-// so could not be forgotten. Children which didn't have virtual entries and
+// forgotten.
-// children with virtual entries will be forgotten even if true is returned.
+func (d *Dir) ForgetAll() {
-func (d *Dir) ForgetAll() (hasVirtual bool) {
+	d.mu.RLock()
-	d.mu.Lock()
+
-	defer d.mu.Unlock()
 	fs.Debugf(d.path, "forgetting directory cache")
 	for _, node := range d.items {
 		if dir, ok := node.(*Dir); ok {
-			if dir.ForgetAll() {
+			dir.ForgetAll()
-				hasVirtual = true
-			}
 		}
 	}
+
+	d.mu.RUnlock()
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
 	// Purge any unnecessary virtual entries
 	d._purgeVirtual()
 
 	d.read = time.Time{}
-	// Check if this dir has virtual entries
+
-	if len(d.virtual) != 0 {
-		hasVirtual = true
-	}
 	// Don't clear directory entries if there are virtual entries in this
 	// directory or any children
-	if !hasVirtual {
+	if d.hasVirtuals() {
-		d.items = make(map[string]Node)
+		d.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
+		return
 	}
-	return hasVirtual
+
+	// Forget the items and stop the timer
+	d.items = make(map[string]Node)
+	d.cleanupTimer.Stop()
 }
 
 // forgetDirPath clears the cache for itself and all subdirectories if
@@ -350,6 +415,9 @@ func (d *Dir) renameTree(dirPath string) {
 // reading everything again
 func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) {
 	d.ForgetAll()
+	virtuals := d.getVirtuals()
+	d.addVirtuals(-virtuals)
+	newParent.addVirtuals(virtuals)
 
 	d.modTimeMu.Lock()
 	d.modTime = fsDir.ModTime(context.TODO())
@@ -386,6 +454,7 @@ func (d *Dir) addObject(node Node) {
 	d.items[leaf] = node
 	if d.virtual == nil {
 		d.virtual = make(map[string]vState)
+		d._addVirtuals(1)
 	}
 	vAdd := vAddFile
 	if node.IsDir() {
@@ -434,6 +503,7 @@ func (d *Dir) delObject(leaf string) {
 	delete(d.items, leaf)
 	if d.virtual == nil {
 		d.virtual = make(map[string]vState)
+		d._addVirtuals(1)
 	}
 	d.virtual[leaf] = vDel
 	fs.Debugf(d.path, "Added virtual directory entry %v: %q", vDel, leaf)
@@ -475,6 +545,8 @@ func (d *Dir) _readDir() error {
 	}
 
 	d.read = when
+	d.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
+
 	return nil
 }
 
@@ -493,6 +565,7 @@ func (d *Dir) _deleteVirtual(name string) {
 	delete(d.virtual, name)
 	if len(d.virtual) == 0 {
 		d.virtual = nil
+		d._addVirtuals(-1)
 	}
 	fs.Debugf(d.path, "Removed virtual directory entry %v: %q", virtualState, name)
 }
@@ -654,6 +727,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
 					dir.read = time.Time{}
 				} else {
 					dir.read = when
+					dir.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
 				}
 				dir.mu.Unlock()
 				if err != nil {
@@ -691,6 +765,7 @@ func (d *Dir) readDirTree() error {
 	}
 	fs.Debugf(d.path, "Reading directory tree done in %s", time.Since(when))
 	d.read = when
+	d.cleanupTimer.Reset(d.vfs.Opt.DirCacheTime * 2)
 	return nil
 }