jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-06 00:03:32 +00:00
Code Issues Projects Releases Wiki Activity

union: add :writback to act as a simple cache

Browse Source 
This adds a :writeback tag to upstreams. If set on a single upstream
then it writes back objects not found into that upstream.

Fixes #6934
This commit is contained in:
Nick Craig-Wood
2023-07-22 08:23:57 +01:00
parent b1c0ae5e7d
commit b33140ddeb
4 changed files with 128 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
backend/union/entry.go
View File

@@ -17,8 +17,9 @@ import (
// This is a wrapped object which returns the Union Fs as its parent
type Object struct {
*upstream.Object
fs *Fs // what this object is part of
co []upstream.Entry
fs *Fs // what this object is part of
co []upstream.Entry
writebackMu sync.Mutex
}
// Directory describes a union Directory
@@ -34,6 +35,13 @@ type entry interface {
candidates() []upstream.Entry
}
// Update o with the contents of newO excluding the lock
func (o *Object) update(newO *Object) {
o.Object = newO.Object
o.fs = newO.fs
o.co = newO.co
}
// UnWrapUpstream returns the upstream Object that this Object is wrapping
func (o *Object) UnWrapUpstream() *upstream.Object {
return o.Object
@@ -67,7 +75,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return err
}
// Update current object
*o = *newO.(*Object)
o.update(newO.(*Object))
return nil
} else if err != nil {
return err
@@ -175,6 +183,25 @@ func (o *Object) SetTier(tier string) error {
return do.SetTier(tier)
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
// Need some sort of locking to prevent multiple downloads
o.writebackMu.Lock()
defer o.writebackMu.Unlock()
// FIXME what if correct object is already in o.co
newObj, err := o.Object.Writeback(ctx)
if err != nil {
return nil, err
}
if newObj != nil {
o.Object = newObj
o.co = append(o.co, newObj) // FIXME should this append or overwrite or update?
}
return o.Object.Object.Open(ctx, options...)
}
// ModTime returns the modification date of the directory
// It returns the latest ModTime of all candidates
func (d *Directory) ModTime(ctx context.Context) (t time.Time) {