mirror of
https://github.com/rclone/rclone.git
synced 2025-12-06 00:03:32 +00:00
azureblob: add metadata and tags support across upload and copy paths
This change adds first-class metadata support to the Azure Blob backend, including headers, user metadata, tags, and modtime overrides, and wires it through uploads and server-side copies. There is a behavior change in that rclone will now set the "mtime" custom metadata when doing server side copies to azure and the `--metadata` argument is given. - Map standard headers: cache-control, content-disposition, content-encoding, content-language, content-type to corresponding x-ms-blob-* HTTP headers. - Map user metadata: any non-reserved keys (excluding x-ms-*) are sent as blob user metadata. Keys are normalized to lowercase for consistency. - Support tags: parse `x-ms-tags` as a comma-separated list of key=value pairs and apply them on uploads and copies. - Support mtime override: accept `mtime` in metadata (RFC3339/RFC3339Nano) to override the stored modtime persisted in user metadata.
This commit is contained in:
@@ -86,12 +86,56 @@ var (
|
|||||||
metadataMu sync.Mutex
|
metadataMu sync.Mutex
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// system metadata keys which this backend owns
|
||||||
|
var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||||
|
"cache-control": {
|
||||||
|
Help: "Cache-Control header",
|
||||||
|
Type: "string",
|
||||||
|
Example: "no-cache",
|
||||||
|
},
|
||||||
|
"content-disposition": {
|
||||||
|
Help: "Content-Disposition header",
|
||||||
|
Type: "string",
|
||||||
|
Example: "inline",
|
||||||
|
},
|
||||||
|
"content-encoding": {
|
||||||
|
Help: "Content-Encoding header",
|
||||||
|
Type: "string",
|
||||||
|
Example: "gzip",
|
||||||
|
},
|
||||||
|
"content-language": {
|
||||||
|
Help: "Content-Language header",
|
||||||
|
Type: "string",
|
||||||
|
Example: "en-US",
|
||||||
|
},
|
||||||
|
"content-type": {
|
||||||
|
Help: "Content-Type header",
|
||||||
|
Type: "string",
|
||||||
|
Example: "text/plain",
|
||||||
|
},
|
||||||
|
"tier": {
|
||||||
|
Help: "Tier of the object",
|
||||||
|
Type: "string",
|
||||||
|
Example: "Hot",
|
||||||
|
ReadOnly: true,
|
||||||
|
},
|
||||||
|
"mtime": {
|
||||||
|
Help: "Time of last modification, read from rclone metadata",
|
||||||
|
Type: "RFC 3339",
|
||||||
|
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// Register with Fs
|
// Register with Fs
|
||||||
func init() {
|
func init() {
|
||||||
fs.Register(&fs.RegInfo{
|
fs.Register(&fs.RegInfo{
|
||||||
Name: "azureblob",
|
Name: "azureblob",
|
||||||
Description: "Microsoft Azure Blob Storage",
|
Description: "Microsoft Azure Blob Storage",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
|
MetadataInfo: &fs.MetadataInfo{
|
||||||
|
System: systemMetadataInfo,
|
||||||
|
Help: `User metadata is stored as x-ms-meta- keys. Azure metadata keys are case insensitive and are always returned in lower case.`,
|
||||||
|
},
|
||||||
Options: []fs.Option{{
|
Options: []fs.Option{{
|
||||||
Name: "account",
|
Name: "account",
|
||||||
Help: `Azure Storage Account Name.
|
Help: `Azure Storage Account Name.
|
||||||
@@ -810,6 +854,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||||||
f.features = (&fs.Features{
|
f.features = (&fs.Features{
|
||||||
ReadMimeType: true,
|
ReadMimeType: true,
|
||||||
WriteMimeType: true,
|
WriteMimeType: true,
|
||||||
|
ReadMetadata: true,
|
||||||
|
WriteMetadata: true,
|
||||||
|
UserMetadata: true,
|
||||||
BucketBased: true,
|
BucketBased: true,
|
||||||
BucketBasedRootOK: true,
|
BucketBasedRootOK: true,
|
||||||
SetTier: true,
|
SetTier: true,
|
||||||
@@ -1157,6 +1204,289 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
|
|||||||
o.meta[modTimeKey] = modTime.Format(timeFormatOut)
|
o.meta[modTimeKey] = modTime.Format(timeFormatOut)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseXMsTags parses the value of the x-ms-tags header into a map.
|
||||||
|
// It expects comma-separated key=value pairs. Whitespace around keys and
|
||||||
|
// values is trimmed. Empty pairs and empty keys are rejected.
|
||||||
|
func parseXMsTags(s string) (map[string]string, error) {
|
||||||
|
if strings.TrimSpace(s) == "" {
|
||||||
|
return map[string]string{}, nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string)
|
||||||
|
parts := strings.Split(s, ",")
|
||||||
|
for _, p := range parts {
|
||||||
|
p = strings.TrimSpace(p)
|
||||||
|
if p == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kv := strings.SplitN(p, "=", 2)
|
||||||
|
if len(kv) != 2 {
|
||||||
|
return nil, fmt.Errorf("invalid tag %q", p)
|
||||||
|
}
|
||||||
|
k := strings.TrimSpace(kv[0])
|
||||||
|
v := strings.TrimSpace(kv[1])
|
||||||
|
if k == "" {
|
||||||
|
return nil, fmt.Errorf("invalid tag key in %q", p)
|
||||||
|
}
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mapMetadataToAzure maps a generic metadata map to Azure HTTP headers,
|
||||||
|
// user metadata, tags and optional modTime override.
|
||||||
|
// Reserved x-ms-* keys (except x-ms-tags) are ignored for user metadata.
|
||||||
|
//
|
||||||
|
// Pass a logger to surface non-fatal parsing issues (e.g. bad mtime).
|
||||||
|
func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (headers blob.HTTPHeaders, userMeta map[string]*string, tags map[string]string, modTime *time.Time, err error) {
|
||||||
|
if meta == nil {
|
||||||
|
return headers, nil, nil, nil, nil
|
||||||
|
}
|
||||||
|
tmp := make(map[string]string)
|
||||||
|
for k, v := range meta {
|
||||||
|
lowerKey := strings.ToLower(k)
|
||||||
|
switch lowerKey {
|
||||||
|
case "cache-control":
|
||||||
|
headers.BlobCacheControl = pString(v)
|
||||||
|
case "content-disposition":
|
||||||
|
headers.BlobContentDisposition = pString(v)
|
||||||
|
case "content-encoding":
|
||||||
|
headers.BlobContentEncoding = pString(v)
|
||||||
|
case "content-language":
|
||||||
|
headers.BlobContentLanguage = pString(v)
|
||||||
|
case "content-type":
|
||||||
|
headers.BlobContentType = pString(v)
|
||||||
|
case "x-ms-tags":
|
||||||
|
parsed, perr := parseXMsTags(v)
|
||||||
|
if perr != nil {
|
||||||
|
return headers, nil, nil, nil, perr
|
||||||
|
}
|
||||||
|
// allocate only if there are tags
|
||||||
|
if len(parsed) > 0 {
|
||||||
|
tags = parsed
|
||||||
|
}
|
||||||
|
case "mtime":
|
||||||
|
// Accept multiple layouts for tolerance
|
||||||
|
var parsed time.Time
|
||||||
|
var pErr error
|
||||||
|
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, timeFormatOut} {
|
||||||
|
parsed, pErr = time.Parse(layout, v)
|
||||||
|
if pErr == nil {
|
||||||
|
modTime = &parsed
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Log and ignore if unparseable
|
||||||
|
if modTime == nil && logf != nil {
|
||||||
|
logf("metadata: couldn't parse mtime %q: %v", v, pErr)
|
||||||
|
}
|
||||||
|
case "tier":
|
||||||
|
// ignore - handled elsewhere
|
||||||
|
default:
|
||||||
|
// Filter out other reserved headers so they don't end up as user metadata
|
||||||
|
if strings.HasPrefix(lowerKey, "x-ms-") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tmp[lowerKey] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
userMeta = toAzureMetaPtr(tmp)
|
||||||
|
return headers, userMeta, tags, modTime, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// toAzureMetaPtr converts a map[string]string to map[string]*string as used by Azure SDK
|
||||||
|
func toAzureMetaPtr(in map[string]string) map[string]*string {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]*string, len(in))
|
||||||
|
for k, v := range in {
|
||||||
|
vv := v
|
||||||
|
out[k] = &vv
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// assembleCopyParams prepares headers, metadata and tags for copy operations.
|
||||||
|
//
|
||||||
|
// It starts from the source properties, optionally overlays mapped metadata
|
||||||
|
// from rclone's metadata options, ensures mtime presence when mapping is
|
||||||
|
// enabled, and returns whether mapping was actually requested (hadMapping).
|
||||||
|
// assembleCopyParams prepares headers, metadata and tags for copy operations.
|
||||||
|
//
|
||||||
|
// If includeBaseMeta is true, start user metadata from the source's metadata
|
||||||
|
// and overlay mapped values. This matches multipart copy commit behavior.
|
||||||
|
// If false, only include mapped user metadata (no source baseline) which
|
||||||
|
// matches previous singlepart StartCopyFromURL semantics.
|
||||||
|
func assembleCopyParams(ctx context.Context, f *Fs, src fs.Object, srcProps *blob.GetPropertiesResponse, includeBaseMeta bool) (headers blob.HTTPHeaders, meta map[string]*string, tags map[string]string, hadMapping bool, err error) {
|
||||||
|
// Start from source properties
|
||||||
|
headers = blob.HTTPHeaders{
|
||||||
|
BlobCacheControl: srcProps.CacheControl,
|
||||||
|
BlobContentDisposition: srcProps.ContentDisposition,
|
||||||
|
BlobContentEncoding: srcProps.ContentEncoding,
|
||||||
|
BlobContentLanguage: srcProps.ContentLanguage,
|
||||||
|
BlobContentMD5: srcProps.ContentMD5,
|
||||||
|
BlobContentType: srcProps.ContentType,
|
||||||
|
}
|
||||||
|
// Optionally deep copy user metadata pointers from source. Normalise keys to
|
||||||
|
// lower-case to avoid duplicate x-ms-meta headers when we later inject/overlay
|
||||||
|
// metadata (Azure treats keys case-insensitively but Go's http.Header will
|
||||||
|
// join duplicate keys into a comma separated list, which breaks shared-key
|
||||||
|
// signing).
|
||||||
|
if includeBaseMeta && len(srcProps.Metadata) > 0 {
|
||||||
|
meta = make(map[string]*string, len(srcProps.Metadata))
|
||||||
|
for k, v := range srcProps.Metadata {
|
||||||
|
if v != nil {
|
||||||
|
vv := *v
|
||||||
|
meta[strings.ToLower(k)] = &vv
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only consider mapping if metadata pipeline is enabled
|
||||||
|
if fs.GetConfig(ctx).Metadata {
|
||||||
|
mapped, mapErr := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
|
||||||
|
if mapErr != nil {
|
||||||
|
return headers, meta, nil, false, fmt.Errorf("failed to map metadata: %w", mapErr)
|
||||||
|
}
|
||||||
|
if mapped != nil {
|
||||||
|
// Map rclone metadata to Azure shapes
|
||||||
|
mappedHeaders, userMeta, mappedTags, mappedModTime, herr := mapMetadataToAzure(mapped, func(format string, args ...any) { fs.Debugf(f, format, args...) })
|
||||||
|
if herr != nil {
|
||||||
|
return headers, meta, nil, false, fmt.Errorf("metadata mapping: %w", herr)
|
||||||
|
}
|
||||||
|
hadMapping = true
|
||||||
|
// Overlay headers (only non-nil)
|
||||||
|
if mappedHeaders.BlobCacheControl != nil {
|
||||||
|
headers.BlobCacheControl = mappedHeaders.BlobCacheControl
|
||||||
|
}
|
||||||
|
if mappedHeaders.BlobContentDisposition != nil {
|
||||||
|
headers.BlobContentDisposition = mappedHeaders.BlobContentDisposition
|
||||||
|
}
|
||||||
|
if mappedHeaders.BlobContentEncoding != nil {
|
||||||
|
headers.BlobContentEncoding = mappedHeaders.BlobContentEncoding
|
||||||
|
}
|
||||||
|
if mappedHeaders.BlobContentLanguage != nil {
|
||||||
|
headers.BlobContentLanguage = mappedHeaders.BlobContentLanguage
|
||||||
|
}
|
||||||
|
if mappedHeaders.BlobContentType != nil {
|
||||||
|
headers.BlobContentType = mappedHeaders.BlobContentType
|
||||||
|
}
|
||||||
|
// Overlay user metadata
|
||||||
|
if len(userMeta) > 0 {
|
||||||
|
if meta == nil {
|
||||||
|
meta = make(map[string]*string, len(userMeta))
|
||||||
|
}
|
||||||
|
for k, v := range userMeta {
|
||||||
|
meta[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Apply tags if any
|
||||||
|
if len(mappedTags) > 0 {
|
||||||
|
tags = mappedTags
|
||||||
|
}
|
||||||
|
// Ensure mtime present using mapped or source time
|
||||||
|
if _, ok := meta[modTimeKey]; !ok {
|
||||||
|
when := src.ModTime(ctx)
|
||||||
|
if mappedModTime != nil {
|
||||||
|
when = *mappedModTime
|
||||||
|
}
|
||||||
|
val := when.Format(time.RFC3339Nano)
|
||||||
|
if meta == nil {
|
||||||
|
meta = make(map[string]*string, 1)
|
||||||
|
}
|
||||||
|
meta[modTimeKey] = &val
|
||||||
|
}
|
||||||
|
// Ensure content-type fallback to source if not set by mapper
|
||||||
|
if headers.BlobContentType == nil {
|
||||||
|
headers.BlobContentType = srcProps.ContentType
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Mapping enabled but not provided: ensure mtime present based on source ModTime
|
||||||
|
if _, ok := meta[modTimeKey]; !ok {
|
||||||
|
when := src.ModTime(ctx)
|
||||||
|
val := when.Format(time.RFC3339Nano)
|
||||||
|
if meta == nil {
|
||||||
|
meta = make(map[string]*string, 1)
|
||||||
|
}
|
||||||
|
meta[modTimeKey] = &val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return headers, meta, tags, hadMapping, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyMappedMetadata applies mapped metadata and headers to the object state for uploads.
|
||||||
|
//
|
||||||
|
// It reads `--metadata`, `--metadata-set`, and `--metadata-mapper` outputs via fs.GetMetadataOptions
|
||||||
|
// and updates o.meta, o.tags and ui.httpHeaders accordingly.
|
||||||
|
func (o *Object) applyMappedMetadata(ctx context.Context, src fs.ObjectInfo, ui *uploadInfo, options []fs.OpenOption) (modTime time.Time, err error) {
|
||||||
|
// Start from the source modtime; may be overridden by metadata
|
||||||
|
modTime = src.ModTime(ctx)
|
||||||
|
|
||||||
|
// Fetch mapped metadata if --metadata is enabled
|
||||||
|
meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
|
if err != nil {
|
||||||
|
return modTime, err
|
||||||
|
}
|
||||||
|
if meta == nil {
|
||||||
|
// No metadata processing requested
|
||||||
|
return modTime, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map metadata using common helper
|
||||||
|
headers, userMeta, tags, mappedModTime, err := mapMetadataToAzure(meta, func(format string, args ...any) { fs.Debugf(o, format, args...) })
|
||||||
|
if err != nil {
|
||||||
|
return modTime, err
|
||||||
|
}
|
||||||
|
// Merge headers into ui
|
||||||
|
if headers.BlobCacheControl != nil {
|
||||||
|
ui.httpHeaders.BlobCacheControl = headers.BlobCacheControl
|
||||||
|
}
|
||||||
|
if headers.BlobContentDisposition != nil {
|
||||||
|
ui.httpHeaders.BlobContentDisposition = headers.BlobContentDisposition
|
||||||
|
}
|
||||||
|
if headers.BlobContentEncoding != nil {
|
||||||
|
ui.httpHeaders.BlobContentEncoding = headers.BlobContentEncoding
|
||||||
|
}
|
||||||
|
if headers.BlobContentLanguage != nil {
|
||||||
|
ui.httpHeaders.BlobContentLanguage = headers.BlobContentLanguage
|
||||||
|
}
|
||||||
|
if headers.BlobContentType != nil {
|
||||||
|
ui.httpHeaders.BlobContentType = headers.BlobContentType
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply user metadata to o.meta with a single critical section
|
||||||
|
if len(userMeta) > 0 {
|
||||||
|
metadataMu.Lock()
|
||||||
|
if o.meta == nil {
|
||||||
|
o.meta = make(map[string]string, len(userMeta))
|
||||||
|
}
|
||||||
|
for k, v := range userMeta {
|
||||||
|
if v != nil {
|
||||||
|
o.meta[k] = *v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
metadataMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply tags
|
||||||
|
if len(tags) > 0 {
|
||||||
|
if o.tags == nil {
|
||||||
|
o.tags = make(map[string]string, len(tags))
|
||||||
|
}
|
||||||
|
for k, v := range tags {
|
||||||
|
o.tags[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if mappedModTime != nil {
|
||||||
|
modTime = *mappedModTime
|
||||||
|
}
|
||||||
|
|
||||||
|
return modTime, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Returns whether file is a directory marker or not
|
// Returns whether file is a directory marker or not
|
||||||
func isDirectoryMarker(size int64, metadata map[string]*string, remote string) bool {
|
func isDirectoryMarker(size int64, metadata map[string]*string, remote string) bool {
|
||||||
// Directory markers are 0 length
|
// Directory markers are 0 length
|
||||||
@@ -1951,18 +2281,19 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert metadata from source object
|
// Prepare metadata/headers/tags for destination
|
||||||
|
// For multipart commit, include base metadata from source then overlay mapped
|
||||||
|
commitHeaders, commitMeta, commitTags, _, err := assembleCopyParams(ctx, f, src, srcProperties, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("multipart copy: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert metadata from source or mapper
|
||||||
options := blockblob.CommitBlockListOptions{
|
options := blockblob.CommitBlockListOptions{
|
||||||
Metadata: srcProperties.Metadata,
|
Metadata: commitMeta,
|
||||||
|
Tags: commitTags,
|
||||||
Tier: parseTier(f.opt.AccessTier),
|
Tier: parseTier(f.opt.AccessTier),
|
||||||
HTTPHeaders: &blob.HTTPHeaders{
|
HTTPHeaders: &commitHeaders,
|
||||||
BlobCacheControl: srcProperties.CacheControl,
|
|
||||||
BlobContentDisposition: srcProperties.ContentDisposition,
|
|
||||||
BlobContentEncoding: srcProperties.ContentEncoding,
|
|
||||||
BlobContentLanguage: srcProperties.ContentLanguage,
|
|
||||||
BlobContentMD5: srcProperties.ContentMD5,
|
|
||||||
BlobContentType: srcProperties.ContentType,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finalise the upload session
|
// Finalise the upload session
|
||||||
@@ -1993,10 +2324,36 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
|
|||||||
return nil, fmt.Errorf("single part copy: source auth: %w", err)
|
return nil, fmt.Errorf("single part copy: source auth: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start the copy
|
// Prepare mapped metadata/tags/headers if requested
|
||||||
options := blob.StartCopyFromURLOptions{
|
options := blob.StartCopyFromURLOptions{
|
||||||
Tier: parseTier(f.opt.AccessTier),
|
Tier: parseTier(f.opt.AccessTier),
|
||||||
}
|
}
|
||||||
|
var postHeaders *blob.HTTPHeaders
|
||||||
|
// Read source properties and assemble params; this also handles the case when mapping is disabled
|
||||||
|
srcProps, err := src.readMetaDataAlways(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("single part copy: read source properties: %w", err)
|
||||||
|
}
|
||||||
|
// For singlepart copy, do not include base metadata from source in StartCopyFromURL
|
||||||
|
headers, meta, tags, hadMapping, aerr := assembleCopyParams(ctx, f, src, srcProps, false)
|
||||||
|
if aerr != nil {
|
||||||
|
return nil, fmt.Errorf("single part copy: %w", aerr)
|
||||||
|
}
|
||||||
|
// Apply tags and post-copy headers only when mapping requested changes
|
||||||
|
if len(tags) > 0 {
|
||||||
|
options.BlobTags = make(map[string]string, len(tags))
|
||||||
|
for k, v := range tags {
|
||||||
|
options.BlobTags[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hadMapping {
|
||||||
|
// Only set metadata explicitly when mapping was requested; otherwise
|
||||||
|
// let the service copy source metadata (including mtime) automatically.
|
||||||
|
if len(meta) > 0 {
|
||||||
|
options.Metadata = meta
|
||||||
|
}
|
||||||
|
postHeaders = &headers
|
||||||
|
}
|
||||||
var startCopy blob.StartCopyFromURLResponse
|
var startCopy blob.StartCopyFromURLResponse
|
||||||
err = f.pacer.Call(func() (bool, error) {
|
err = f.pacer.Call(func() (bool, error) {
|
||||||
startCopy, err = dstBlobSVC.StartCopyFromURL(ctx, srcURL, &options)
|
startCopy, err = dstBlobSVC.StartCopyFromURL(ctx, srcURL, &options)
|
||||||
@@ -2026,6 +2383,16 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
|
|||||||
pollTime = min(2*pollTime, time.Second)
|
pollTime = min(2*pollTime, time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If mapper requested header changes, set them post-copy
|
||||||
|
if postHeaders != nil {
|
||||||
|
blb := f.getBlobSVC(dstContainer, dstPath)
|
||||||
|
_, setErr := blb.SetHTTPHeaders(ctx, *postHeaders, nil)
|
||||||
|
if setErr != nil {
|
||||||
|
return nil, fmt.Errorf("single part copy: failed to set headers: %w", setErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Metadata (when requested) is set via StartCopyFromURL options.Metadata
|
||||||
|
|
||||||
return f.NewObject(ctx, remote)
|
return f.NewObject(ctx, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2157,6 +2524,35 @@ func (o *Object) getMetadata() (metadata map[string]*string) {
|
|||||||
return metadata
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Metadata returns metadata for an object
|
||||||
|
//
|
||||||
|
// It returns a combined view of system and user metadata.
|
||||||
|
func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
|
||||||
|
// Ensure metadata is loaded
|
||||||
|
if err := o.readMetaData(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
m := fs.Metadata{}
|
||||||
|
|
||||||
|
// System metadata we expose
|
||||||
|
if !o.modTime.IsZero() {
|
||||||
|
m["mtime"] = o.modTime.Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
if o.accessTier != "" {
|
||||||
|
m["tier"] = string(o.accessTier)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge user metadata (already lower-cased keys)
|
||||||
|
metadataMu.Lock()
|
||||||
|
for k, v := range o.meta {
|
||||||
|
m[k] = v
|
||||||
|
}
|
||||||
|
metadataMu.Unlock()
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
// decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
|
// decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
|
||||||
//
|
//
|
||||||
// Sets
|
// Sets
|
||||||
@@ -2995,17 +3391,19 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
|||||||
// containerPath = containerPath[:len(containerPath)-1]
|
// containerPath = containerPath[:len(containerPath)-1]
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// Update Mod time
|
// Start with default content-type based on source
|
||||||
o.updateMetadataWithModTime(src.ModTime(ctx))
|
|
||||||
if err != nil {
|
|
||||||
return ui, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create the HTTP headers for the upload
|
|
||||||
ui.httpHeaders = blob.HTTPHeaders{
|
ui.httpHeaders = blob.HTTPHeaders{
|
||||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Apply mapped metadata/headers/tags if requested
|
||||||
|
modTime, err := o.applyMappedMetadata(ctx, src, &ui, options)
|
||||||
|
if err != nil {
|
||||||
|
return ui, err
|
||||||
|
}
|
||||||
|
// Ensure mtime is set in metadata based on possibly overridden modTime
|
||||||
|
o.updateMetadataWithModTime(modTime)
|
||||||
|
|
||||||
// Compute the Content-MD5 of the file. As we stream all uploads it
|
// Compute the Content-MD5 of the file. As we stream all uploads it
|
||||||
// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
|
// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
|
||||||
if !o.fs.opt.DisableCheckSum {
|
if !o.fs.opt.DisableCheckSum {
|
||||||
|
|||||||
@@ -5,11 +5,16 @@ package azureblob
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
|
"github.com/rclone/rclone/fs/object"
|
||||||
"github.com/rclone/rclone/fstest"
|
"github.com/rclone/rclone/fstest"
|
||||||
"github.com/rclone/rclone/fstest/fstests"
|
"github.com/rclone/rclone/fstest/fstests"
|
||||||
"github.com/rclone/rclone/lib/random"
|
"github.com/rclone/rclone/lib/random"
|
||||||
@@ -148,4 +153,417 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
|||||||
func (f *Fs) InternalTest(t *testing.T) {
|
func (f *Fs) InternalTest(t *testing.T) {
|
||||||
t.Run("Features", f.testFeatures)
|
t.Run("Features", f.testFeatures)
|
||||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||||
|
t.Run("Metadata", f.testMetadataPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
// helper to read blob properties for an object
|
||||||
|
func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
|
||||||
|
ao := o.(*Object)
|
||||||
|
props, err := ao.readMetaDataAlways(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return props
|
||||||
|
}
|
||||||
|
|
||||||
|
// helper to assert select headers and user metadata
|
||||||
|
func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
|
||||||
|
// Headers
|
||||||
|
get := func(p *string) string {
|
||||||
|
if p == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *p
|
||||||
|
}
|
||||||
|
if v, ok := want["content-type"]; ok {
|
||||||
|
assert.Equal(t, v, get(props.ContentType), "content-type")
|
||||||
|
}
|
||||||
|
if v, ok := want["cache-control"]; ok {
|
||||||
|
assert.Equal(t, v, get(props.CacheControl), "cache-control")
|
||||||
|
}
|
||||||
|
if v, ok := want["content-disposition"]; ok {
|
||||||
|
assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
|
||||||
|
}
|
||||||
|
if v, ok := want["content-encoding"]; ok {
|
||||||
|
assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
|
||||||
|
}
|
||||||
|
if v, ok := want["content-language"]; ok {
|
||||||
|
assert.Equal(t, v, get(props.ContentLanguage), "content-language")
|
||||||
|
}
|
||||||
|
// User metadata (case-insensitive keys from service)
|
||||||
|
norm := make(map[string]*string, len(props.Metadata))
|
||||||
|
for kk, vv := range props.Metadata {
|
||||||
|
norm[strings.ToLower(kk)] = vv
|
||||||
|
}
|
||||||
|
for k, v := range wantUserMeta {
|
||||||
|
pv, ok := norm[strings.ToLower(k)]
|
||||||
|
if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
|
||||||
|
if pv == nil {
|
||||||
|
assert.Equal(t, v, "", k)
|
||||||
|
} else {
|
||||||
|
assert.Equal(t, v, *pv, k)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Log available keys for diagnostics
|
||||||
|
keys := make([]string, 0, len(props.Metadata))
|
||||||
|
for kk := range props.Metadata {
|
||||||
|
keys = append(keys, kk)
|
||||||
|
}
|
||||||
|
t.Logf("available user metadata keys: %v", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// helper to read blob tags for an object
|
||||||
|
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||||
|
ao := o.(*Object)
|
||||||
|
blb := ao.getBlobSVC()
|
||||||
|
resp, err := blb.GetTags(ctx, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
out := make(map[string]string)
|
||||||
|
for _, tag := range resp.BlobTagSet {
|
||||||
|
if tag.Key != nil {
|
||||||
|
k := *tag.Key
|
||||||
|
v := ""
|
||||||
|
if tag.Value != nil {
|
||||||
|
v = *tag.Value
|
||||||
|
}
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test metadata across different write paths
|
||||||
|
func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Common expected metadata and headers
|
||||||
|
baseMeta := fs.Metadata{
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"content-disposition": "inline",
|
||||||
|
"content-language": "en-US",
|
||||||
|
// Note: Don't set content-encoding here to avoid download decoding differences
|
||||||
|
// We will set a custom user metadata key
|
||||||
|
"potato": "royal",
|
||||||
|
// and modtime
|
||||||
|
"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Singlepart upload
|
||||||
|
t.Run("PutSinglepart", func(t *testing.T) {
|
||||||
|
// size less than chunk size
|
||||||
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||||
|
item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
// override content-type via metadata mapping
|
||||||
|
meta := fs.Metadata{}
|
||||||
|
meta.Merge(baseMeta)
|
||||||
|
meta["content-type"] = "text/plain"
|
||||||
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||||
|
defer func() { _ = obj.Remove(ctx) }()
|
||||||
|
|
||||||
|
props := getProps(ctx, t, obj)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"content-disposition": "inline",
|
||||||
|
"content-language": "en-US",
|
||||||
|
}, map[string]string{
|
||||||
|
"potato": "royal",
|
||||||
|
})
|
||||||
|
_ = http.StatusOK // keep import for parity but don't inspect RawResponse
|
||||||
|
})
|
||||||
|
|
||||||
|
// Multipart upload
|
||||||
|
t.Run("PutMultipart", func(t *testing.T) {
|
||||||
|
// size greater than chunk size to force multipart
|
||||||
|
contents := random.String(int(f.opt.ChunkSize + 1024))
|
||||||
|
item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
meta := fs.Metadata{}
|
||||||
|
meta.Merge(baseMeta)
|
||||||
|
meta["content-type"] = "application/json"
|
||||||
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||||
|
defer func() { _ = obj.Remove(ctx) }()
|
||||||
|
|
||||||
|
props := getProps(ctx, t, obj)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "application/json",
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"content-disposition": "inline",
|
||||||
|
"content-language": "en-US",
|
||||||
|
}, map[string]string{
|
||||||
|
"potato": "royal",
|
||||||
|
})
|
||||||
|
|
||||||
|
// Tags: Singlepart upload
|
||||||
|
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||||
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||||
|
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
meta := fs.Metadata{
|
||||||
|
"x-ms-tags": "env=dev,team=sync",
|
||||||
|
}
|
||||||
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
|
||||||
|
defer func() { _ = obj.Remove(ctx) }()
|
||||||
|
|
||||||
|
tags := getTagsMap(ctx, t, obj)
|
||||||
|
assert.Equal(t, "dev", tags["env"])
|
||||||
|
assert.Equal(t, "sync", tags["team"])
|
||||||
|
})
|
||||||
|
|
||||||
|
// Tags: Multipart upload
|
||||||
|
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||||
|
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||||
|
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
meta := fs.Metadata{
|
||||||
|
"x-ms-tags": "project=alpha,release=2025-08",
|
||||||
|
}
|
||||||
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
|
||||||
|
defer func() { _ = obj.Remove(ctx) }()
|
||||||
|
|
||||||
|
tags := getTagsMap(ctx, t, obj)
|
||||||
|
assert.Equal(t, "alpha", tags["project"])
|
||||||
|
assert.Equal(t, "2025-08", tags["release"])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||||
|
t.Run("CopySinglepart", func(t *testing.T) {
|
||||||
|
// create small source
|
||||||
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||||
|
srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
// set mapping via MetadataSet
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
ci.MetadataSet = fs.Metadata{
|
||||||
|
"cache-control": "private, max-age=60",
|
||||||
|
"content-disposition": "attachment; filename=foo.txt",
|
||||||
|
"content-language": "fr",
|
||||||
|
// no content-type: should fallback to source
|
||||||
|
"potato": "maris",
|
||||||
|
}
|
||||||
|
|
||||||
|
// do copy
|
||||||
|
dstName := "meta-copy-single-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
props := getProps(ctx2, t, dst)
|
||||||
|
// content-type should fallback to source (text/plain)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "text/plain",
|
||||||
|
"cache-control": "private, max-age=60",
|
||||||
|
"content-disposition": "attachment; filename=foo.txt",
|
||||||
|
"content-language": "fr",
|
||||||
|
}, map[string]string{
|
||||||
|
"potato": "maris",
|
||||||
|
})
|
||||||
|
// mtime should be populated on copy when --metadata is used
|
||||||
|
// and should equal the source ModTime (RFC3339Nano)
|
||||||
|
// Read user metadata (case-insensitive)
|
||||||
|
m := props.Metadata
|
||||||
|
var gotMtime string
|
||||||
|
for k, v := range m {
|
||||||
|
if strings.EqualFold(k, "mtime") && v != nil {
|
||||||
|
gotMtime = *v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||||
|
// parse and compare times ignoring formatting differences
|
||||||
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
|
||||||
|
t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
|
||||||
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||||
|
srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
|
||||||
|
dstName := "meta-copy-single-only-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
props := getProps(ctx2, t, dst)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "text/plain",
|
||||||
|
}, map[string]string{})
|
||||||
|
// Assert mtime injected
|
||||||
|
m := props.Metadata
|
||||||
|
var gotMtime string
|
||||||
|
for k, v := range m {
|
||||||
|
if strings.EqualFold(k, "mtime") && v != nil {
|
||||||
|
gotMtime = *v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||||
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||||
|
t.Run("CopyMultipart", func(t *testing.T) {
|
||||||
|
// create large source to force multipart
|
||||||
|
contents := random.String(int(f.opt.CopyCutoff + 1024))
|
||||||
|
srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
// set mapping via MetadataSet
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
ci.MetadataSet = fs.Metadata{
|
||||||
|
"cache-control": "max-age=0, no-cache",
|
||||||
|
// omit content-type to trigger fallback
|
||||||
|
"content-language": "de",
|
||||||
|
"potato": "desiree",
|
||||||
|
}
|
||||||
|
|
||||||
|
dstName := "meta-copy-multi-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
props := getProps(ctx2, t, dst)
|
||||||
|
// content-type should fallback to source (application/octet-stream)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "application/octet-stream",
|
||||||
|
"cache-control": "max-age=0, no-cache",
|
||||||
|
"content-language": "de",
|
||||||
|
}, map[string]string{
|
||||||
|
"potato": "desiree",
|
||||||
|
})
|
||||||
|
// mtime should be populated on copy when --metadata is used
|
||||||
|
m := props.Metadata
|
||||||
|
var gotMtime string
|
||||||
|
for k, v := range m {
|
||||||
|
if strings.EqualFold(k, "mtime") && v != nil {
|
||||||
|
gotMtime = *v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||||
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// CopyMultipart with only --metadata must inject mtime and preserve src content-type
|
||||||
|
t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
|
||||||
|
contents := random.String(int(f.opt.CopyCutoff + 2048))
|
||||||
|
srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
|
||||||
|
dstName := "meta-copy-multi-only-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
props := getProps(ctx2, t, dst)
|
||||||
|
assertHeadersAndMetadata(t, props, map[string]string{
|
||||||
|
"content-type": "application/octet-stream",
|
||||||
|
}, map[string]string{})
|
||||||
|
m := props.Metadata
|
||||||
|
var gotMtime string
|
||||||
|
for k, v := range m {
|
||||||
|
if strings.EqualFold(k, "mtime") && v != nil {
|
||||||
|
gotMtime = *v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||||
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Tags: Singlepart copy
|
||||||
|
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||||
|
// create small source
|
||||||
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||||
|
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
// set mapping via MetadataSet including tags
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
ci.MetadataSet = fs.Metadata{
|
||||||
|
"x-ms-tags": "copy=single,mode=test",
|
||||||
|
}
|
||||||
|
|
||||||
|
dstName := "tags-copy-single-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
tags := getTagsMap(ctx2, t, dst)
|
||||||
|
assert.Equal(t, "single", tags["copy"])
|
||||||
|
assert.Equal(t, "test", tags["mode"])
|
||||||
|
})
|
||||||
|
|
||||||
|
// Tags: Multipart copy
|
||||||
|
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||||
|
// create large source to force multipart
|
||||||
|
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||||
|
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||||
|
defer func() { _ = srcObj.Remove(ctx) }()
|
||||||
|
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
ci.MetadataSet = fs.Metadata{
|
||||||
|
"x-ms-tags": "copy=multi,mode=test",
|
||||||
|
}
|
||||||
|
|
||||||
|
dstName := "tags-copy-multi-dst.txt"
|
||||||
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer func() { _ = dst.Remove(ctx2) }()
|
||||||
|
|
||||||
|
tags := getTagsMap(ctx2, t, dst)
|
||||||
|
assert.Equal(t, "multi", tags["copy"])
|
||||||
|
assert.Equal(t, "test", tags["mode"])
|
||||||
|
})
|
||||||
|
|
||||||
|
// Negative: invalid x-ms-tags must error
|
||||||
|
t.Run("InvalidXMsTags", func(t *testing.T) {
|
||||||
|
contents := random.String(32)
|
||||||
|
item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||||
|
// construct ObjectInfo with invalid x-ms-tags
|
||||||
|
buf := strings.NewReader(contents)
|
||||||
|
// Build obj info with metadata
|
||||||
|
meta := fs.Metadata{
|
||||||
|
"x-ms-tags": "badpair-without-equals",
|
||||||
|
}
|
||||||
|
// force metadata on
|
||||||
|
ctx2, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
|
||||||
|
obji = obji.WithMetadata(meta).WithMimeType("text/plain")
|
||||||
|
_, err := f.Put(ctx2, buf, obji)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "invalid tag")
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -103,6 +103,26 @@ MD5 hashes are stored with blobs. However blobs that were uploaded in
|
|||||||
chunks only have an MD5 if the source remote was capable of MD5
|
chunks only have an MD5 if the source remote was capable of MD5
|
||||||
hashes, e.g. the local disk.
|
hashes, e.g. the local disk.
|
||||||
|
|
||||||
|
### Metadata and tags
|
||||||
|
|
||||||
|
Rclone can map arbitrary metadata to Azure Blob headers, user metadata, and tags
|
||||||
|
when `--metadata` is enabled (or when using `--metadata-set` / `--metadata-mapper`).
|
||||||
|
|
||||||
|
- Headers: Set these keys in metadata to map to the corresponding blob headers:
|
||||||
|
- `cache-control`, `content-disposition`, `content-encoding`, `content-language`, `content-type`.
|
||||||
|
- User metadata: Any other non-reserved keys are written as user metadata
|
||||||
|
(keys are normalized to lowercase). Keys starting with `x-ms-` are reserved and
|
||||||
|
are not stored as user metadata.
|
||||||
|
- Tags: Provide `x-ms-tags` as a comma-separated list of `key=value` pairs, e.g.
|
||||||
|
`x-ms-tags=env=dev,team=sync`. These are applied as blob tags on upload and on
|
||||||
|
server-side copies. Whitespace around keys/values is ignored.
|
||||||
|
- Modtime override: Provide `mtime` in RFC3339/RFC3339Nano format to override the
|
||||||
|
stored modtime persisted in user metadata. If `mtime` cannot be parsed, rclone
|
||||||
|
logs a debug message and ignores the override.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- Rclone ignores reserved `x-ms-*` keys (except `x-ms-tags`) for user metadata.
|
||||||
|
|
||||||
### Performance
|
### Performance
|
||||||
|
|
||||||
When uploading large files, increasing the value of
|
When uploading large files, increasing the value of
|
||||||
|
|||||||
@@ -368,7 +368,7 @@ func Run(t *testing.T, opt *Opt) {
|
|||||||
}
|
}
|
||||||
file1Contents string
|
file1Contents string
|
||||||
file1MimeType = "text/csv"
|
file1MimeType = "text/csv"
|
||||||
file1Metadata = fs.Metadata{"rclone-test": "potato"}
|
file1Metadata = fs.Metadata{"rclonetest": "potato"}
|
||||||
file2 = fstest.Item{
|
file2 = fstest.Item{
|
||||||
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
|
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
|
||||||
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
|
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
|
||||||
|
|||||||
Reference in New Issue
Block a user