s3: experiment checking SSE multipart etags - FIXME WIP DO NOT MERGE

This retrieves the ETags from the uploaded parts and combines those
together to make the overall ETag.

This is probably slightly less robust...

See: https://forum.rclone.org/t/in-some-cases-rclone-does-not-use-etag-to-verify-files/36095/8

backend/s3/s3.go

@@ -5160,7 +5160,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 
 			// create checksum of buffer for integrity checking
 			md5sumBinary := md5.Sum(buf)
-			addMd5(&md5sumBinary, partNum-1)
+			//addMd5(&md5sumBinary, partNum-1)
 			md5sum := base64.StdEncoding.EncodeToString(md5sumBinary[:])
 
 			err = f.pacer.Call(func() (bool, error) {
@@ -5191,7 +5191,15 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 					ETag:       uout.ETag,
 				})
 				partsMu.Unlock()
-
+				if uout.ETag != nil {
+					etag := strings.Trim(strings.ToLower(*uout.ETag), `"`)
+					etagBinary, err := hex.DecodeString(etag)
+					if err != nil || len(etagBinary) != md5.Size {
+						fs.Errorf(o, "Failed to decode ETag %q: %v", etag, err)
+					} else {
+						addMd5((*[md5.Size]byte)(etagBinary), partNum-1)
+					}
+				}
 				return false, nil
 			})
 			if err != nil {
@@ -5586,7 +5594,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	o.setMetaData(head)
 
 	// Check multipart upload ETag if required
-	if o.fs.opt.UseMultipartEtag.Value && !o.fs.etagIsNotMD5 && wantETag != "" && head.ETag != nil && *head.ETag != "" {
+	if o.fs.opt.UseMultipartEtag.Value /*&& !o.fs.etagIsNotMD5*/ && wantETag != "" && head.ETag != nil && *head.ETag != "" {
 		gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`)
 		if wantETag != gotETag {
 			return fmt.Errorf("multipart upload corrupted: Etag differ: expecting %s but got %s", wantETag, gotETag)

vfs/file.go

@@ -322,9 +322,7 @@ func (f *File) ModTime() (modTime time.Time) {
 	}
 	// Read the modtime from a dirty item if it exists
 	if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
-		item := f.d.vfs.cache.ItemOrNil(f._path())
-		noModTime := f.d.f.Precision() == fs.ModTimeNotSupported
-		if item != nil && (item.IsDirty() || noModTime) {
+		if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
 			modTime, err := item.GetModTime()
 			if err != nil {
 				fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)

vfs/vfscache/cache.go

@@ -294,22 +294,14 @@ func (c *Cache) put(name string, item *Item) (oldItem *Item) {
 	return oldItem
 }
 
-// ItemOrNil returns the Item if it exists in the cache otherwise it
-// returns nil.
-//
-// name should be a remote path not an osPath
-func (c *Cache) ItemOrNil(name string) (item *Item) {
-	name = clean(name)
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.item[name]
-}
-
 // InUse returns whether the name is in use in the cache
 //
 // name should be a remote path not an osPath
 func (c *Cache) InUse(name string) bool {
-	item := c.ItemOrNil(name)
+	name = clean(name)
+	c.mu.Lock()
+	item := c.item[name]
+	c.mu.Unlock()
 	if item == nil {
 		return false
 	}
@@ -321,7 +313,10 @@ func (c *Cache) InUse(name string) bool {
 //
 // name should be a remote path not an osPath
 func (c *Cache) DirtyItem(name string) (item *Item) {
-	item = c.ItemOrNil(name)
+	name = clean(name)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	item = c.item[name]
 	if item != nil && !item.IsDirty() {
 		item = nil
 	}

vfs/vfscache/item.go

@@ -1223,7 +1223,7 @@ func (item *Item) setModTime(modTime time.Time) {
 	item.mu.Unlock()
 }
 
-// GetModTime of the cache item
+// GetModTime of the cache file
 func (item *Item) GetModTime() (modTime time.Time, err error) {
 	// defer log.Trace(item.name, "modTime=%v", modTime)("")
 	item.mu.Lock()
@@ -1231,9 +1231,6 @@ func (item *Item) GetModTime() (modTime time.Time, err error) {
 	fi, err := item._stat()
 	if err == nil {
 		modTime = fi.ModTime()
-		item.info.ModTime = modTime
-	} else {
-		modTime = item.info.ModTime
 	}
 	return modTime, nil
 }