mirror of
https://github.com/rclone/rclone.git
synced 2026-02-20 19:33:28 +00:00
s3: add Object Lock support
Add support for S3 Object Lock with the following new options: - --s3-object-lock-mode: set retention mode (GOVERNANCE/COMPLIANCE/copy) - --s3-object-lock-retain-until-date: set retention date (RFC3339/duration/copy) - --s3-object-lock-legal-hold-status: set legal hold (ON/OFF/copy) - --s3-bypass-governance-retention: bypass GOVERNANCE lock on delete - --s3-bucket-object-lock-enabled: enable Object Lock on bucket creation - --s3-object-lock-set-after-upload: apply lock via separate API calls The special value "copy" preserves the source object's setting when used with --metadata flag, enabling scenarios like cloning objects from COMPLIANCE to GOVERNANCE mode while preserving the original retention date. Includes integration tests that create a temporary Object Lock bucket covering: - Retention Mode and Date - Legal Hold - Apply settings after upload - Override protections using bypass-governance flag The tests are gracefully skipped on providers that do not support Object Lock. Fixes #4683 Closes #7894 #7893 #8866
This commit is contained in:
379
backend/s3/s3.go
379
backend/s3/s3.go
@@ -828,6 +828,107 @@ use |-vv| to see the debug level logs.
|
||||
}, {
|
||||
Name: "ibm_resource_instance_id",
|
||||
Help: "IBM service instance id",
|
||||
}, {
|
||||
Name: "object_lock_mode",
|
||||
Help: `Object Lock mode to apply when uploading or copying objects.
|
||||
|
||||
Set this to apply Object Lock retention mode to objects.
|
||||
If not set, no Object Lock mode is applied (even with --metadata).
|
||||
|
||||
Note: To enable Object Lock retention, you must set BOTH object_lock_mode
|
||||
AND object_lock_retain_until_date. Setting only one has no effect.
|
||||
|
||||
- GOVERNANCE: Set Object Lock mode to GOVERNANCE
|
||||
- COMPLIANCE: Set Object Lock mode to COMPLIANCE
|
||||
- copy: Copy the mode from the source object (requires --metadata)
|
||||
|
||||
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "GOVERNANCE",
|
||||
Help: "Set Object Lock mode to GOVERNANCE",
|
||||
}, {
|
||||
Value: "COMPLIANCE",
|
||||
Help: "Set Object Lock mode to COMPLIANCE",
|
||||
}, {
|
||||
Value: "copy",
|
||||
Help: "Copy from source object (requires --metadata)",
|
||||
}},
|
||||
}, {
|
||||
Name: "object_lock_retain_until_date",
|
||||
Help: `Object Lock retention until date to apply when uploading or copying objects.
|
||||
|
||||
Set this to apply Object Lock retention date to objects.
|
||||
If not set, no retention date is applied (even with --metadata).
|
||||
|
||||
Note: To enable Object Lock retention, you must set BOTH object_lock_mode
|
||||
AND object_lock_retain_until_date. Setting only one has no effect.
|
||||
|
||||
Accepts:
|
||||
- RFC 3339 format: 2030-01-02T15:04:05Z
|
||||
- Duration from now: 365d, 1y, 6M (days, years, months)
|
||||
- copy: Copy the date from the source object (requires --metadata)
|
||||
|
||||
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "copy",
|
||||
Help: "Copy from source object (requires --metadata)",
|
||||
}, {
|
||||
Value: "2030-01-01T00:00:00Z",
|
||||
Help: "Set specific date (RFC 3339 format)",
|
||||
}, {
|
||||
Value: "365d",
|
||||
Help: "Set retention for 365 days from now",
|
||||
}, {
|
||||
Value: "1y",
|
||||
Help: "Set retention for 1 year from now",
|
||||
}},
|
||||
}, {
|
||||
Name: "object_lock_legal_hold_status",
|
||||
Help: `Object Lock legal hold status to apply when uploading or copying objects.
|
||||
|
||||
Set this to apply Object Lock legal hold to objects.
|
||||
If not set, no legal hold is applied (even with --metadata).
|
||||
|
||||
Note: Legal hold is independent of retention and can be set separately.
|
||||
|
||||
- ON: Enable legal hold
|
||||
- OFF: Disable legal hold
|
||||
- copy: Copy the legal hold status from the source object (requires --metadata)
|
||||
|
||||
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "ON",
|
||||
Help: "Enable legal hold",
|
||||
}, {
|
||||
Value: "OFF",
|
||||
Help: "Disable legal hold",
|
||||
}, {
|
||||
Value: "copy",
|
||||
Help: "Copy from source object (requires --metadata)",
|
||||
}},
|
||||
}, {
|
||||
Name: "bypass_governance_retention",
|
||||
Help: `Allow deleting or modifying objects locked with GOVERNANCE mode.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "bucket_object_lock_enabled",
|
||||
Help: `Enable Object Lock when creating new buckets.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "object_lock_set_after_upload",
|
||||
Help: `Set Object Lock via separate API calls after upload.
|
||||
|
||||
Use this for S3-compatible providers that don't support setting Object Lock
|
||||
headers during PUT operations. When enabled, Object Lock is set via separate
|
||||
PutObjectRetention and PutObjectLegalHold API calls after the upload completes.
|
||||
|
||||
This adds extra API calls per object, so only enable if your provider requires it.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
}}))
|
||||
}
|
||||
@@ -922,6 +1023,21 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"object-lock-mode": {
|
||||
Help: "Object Lock mode: GOVERNANCE or COMPLIANCE",
|
||||
Type: "string",
|
||||
Example: "GOVERNANCE",
|
||||
},
|
||||
"object-lock-retain-until-date": {
|
||||
Help: "Object Lock retention until date",
|
||||
Type: "RFC 3339",
|
||||
Example: "2030-01-02T15:04:05Z",
|
||||
},
|
||||
"object-lock-legal-hold-status": {
|
||||
Help: "Object Lock legal hold status: ON or OFF",
|
||||
Type: "string",
|
||||
Example: "OFF",
|
||||
},
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
@@ -992,6 +1108,12 @@ type Options struct {
|
||||
IBMInstanceID string `config:"ibm_resource_instance_id"`
|
||||
UseXID fs.Tristate `config:"use_x_id"`
|
||||
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
|
||||
ObjectLockMode string `config:"object_lock_mode"`
|
||||
ObjectLockRetainUntilDate string `config:"object_lock_retain_until_date"`
|
||||
ObjectLockLegalHoldStatus string `config:"object_lock_legal_hold_status"`
|
||||
BypassGovernanceRetention bool `config:"bypass_governance_retention"`
|
||||
BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"`
|
||||
ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"`
|
||||
}
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
@@ -1036,6 +1158,11 @@ type Object struct {
|
||||
contentDisposition *string // Content-Disposition: header
|
||||
contentEncoding *string // Content-Encoding: header
|
||||
contentLanguage *string // Content-Language: header
|
||||
|
||||
// Object Lock metadata
|
||||
objectLockMode *string // Object Lock mode: GOVERNANCE or COMPLIANCE
|
||||
objectLockRetainUntilDate *time.Time // Object Lock retention until date
|
||||
objectLockLegalHoldStatus *string // Object Lock legal hold: ON or OFF
|
||||
}
|
||||
|
||||
// safely dereference the pointer, returning a zero T if nil
|
||||
@@ -1056,6 +1183,21 @@ func getHTTPStatusCode(err error) int {
|
||||
return -1
|
||||
}
|
||||
|
||||
// parseRetainUntilDate parses a retain until date from a string.
|
||||
// It accepts RFC 3339 format or duration strings like "365d", "1y", "6m".
|
||||
func parseRetainUntilDate(s string) (time.Time, error) {
|
||||
// First try RFC 3339 format
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
// Try as a duration from now
|
||||
d, err := fs.ParseDuration(s)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("can't parse %q as RFC 3339 date or duration: %w", s, err)
|
||||
}
|
||||
return time.Now().Add(d), nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
@@ -2649,8 +2791,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
|
||||
}
|
||||
return f.cache.Create(bucket, func() error {
|
||||
req := s3.CreateBucketInput{
|
||||
Bucket: &bucket,
|
||||
ACL: types.BucketCannedACL(f.opt.BucketACL),
|
||||
Bucket: &bucket,
|
||||
ACL: types.BucketCannedACL(f.opt.BucketACL),
|
||||
ObjectLockEnabledForBucket: &f.opt.BucketObjectLockEnabled,
|
||||
}
|
||||
if f.opt.LocationConstraint != "" {
|
||||
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
|
||||
@@ -2769,6 +2912,24 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
|
||||
req.StorageClass = types.StorageClass(f.opt.StorageClass)
|
||||
}
|
||||
|
||||
// Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
|
||||
// "copy" means: keep the value from source (passed via req from prepareUpload/setFrom functions)
|
||||
if !f.opt.ObjectLockSetAfterUpload {
|
||||
if f.opt.ObjectLockMode != "" && !strings.EqualFold(f.opt.ObjectLockMode, "copy") {
|
||||
req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(f.opt.ObjectLockMode))
|
||||
}
|
||||
if f.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(f.opt.ObjectLockRetainUntilDate, "copy") {
|
||||
retainDate, err := parseRetainUntilDate(f.opt.ObjectLockRetainUntilDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid object_lock_retain_until_date: %w", err)
|
||||
}
|
||||
req.ObjectLockRetainUntilDate = &retainDate
|
||||
}
|
||||
if f.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(f.opt.ObjectLockLegalHoldStatus, "copy") {
|
||||
req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(f.opt.ObjectLockLegalHoldStatus))
|
||||
}
|
||||
}
|
||||
|
||||
if src.bytes >= int64(f.opt.CopyCutoff) {
|
||||
return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)
|
||||
}
|
||||
@@ -2951,7 +3112,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
|
||||
setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
|
||||
if ci.Metadata {
|
||||
// Use REPLACE directive if metadata is being modified, otherwise S3 ignores our values
|
||||
// This is needed when:
|
||||
// 1. --metadata flag is set
|
||||
// 2. Any Object Lock option is set (to override or explicitly copy)
|
||||
needsReplace := ci.Metadata ||
|
||||
f.opt.ObjectLockMode != "" ||
|
||||
f.opt.ObjectLockRetainUntilDate != "" ||
|
||||
f.opt.ObjectLockLegalHoldStatus != ""
|
||||
if needsReplace {
|
||||
req.MetadataDirective = types.MetadataDirectiveReplace
|
||||
}
|
||||
|
||||
@@ -2959,7 +3128,21 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.NewObject(ctx, remote)
|
||||
dstObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set Object Lock via separate API calls if requested
|
||||
if f.opt.ObjectLockSetAfterUpload {
|
||||
if dstObject, ok := dstObj.(*Object); ok {
|
||||
if err := dstObject.setObjectLockAfterUpload(ctx, srcObj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
@@ -3838,6 +4021,19 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
|
||||
o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
|
||||
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
|
||||
|
||||
// Set Object Lock metadata
|
||||
if resp.ObjectLockMode != "" {
|
||||
mode := string(resp.ObjectLockMode)
|
||||
o.objectLockMode = &mode
|
||||
}
|
||||
if resp.ObjectLockRetainUntilDate != nil {
|
||||
o.objectLockRetainUntilDate = resp.ObjectLockRetainUntilDate
|
||||
}
|
||||
if resp.ObjectLockLegalHoldStatus != "" {
|
||||
status := string(resp.ObjectLockLegalHoldStatus)
|
||||
o.objectLockLegalHoldStatus = &status
|
||||
}
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" {
|
||||
o.bytes = -1
|
||||
@@ -4556,6 +4752,26 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
case "btime":
|
||||
// write as metadata since we can't set it
|
||||
ui.req.Metadata[k] = v
|
||||
case "object-lock-mode":
|
||||
// Only apply if option is set to "copy" and not using after-upload API
|
||||
if strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
|
||||
ui.req.ObjectLockMode = types.ObjectLockMode(v)
|
||||
}
|
||||
case "object-lock-retain-until-date":
|
||||
// Only apply if option is set to "copy" and not using after-upload API
|
||||
if strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
|
||||
retainDate, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "failed to parse object-lock-retain-until-date %q: %v", v, err)
|
||||
} else {
|
||||
ui.req.ObjectLockRetainUntilDate = &retainDate
|
||||
}
|
||||
}
|
||||
case "object-lock-legal-hold-status":
|
||||
// Only apply if option is set to "copy" and not using after-upload API
|
||||
if strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
|
||||
ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(v)
|
||||
}
|
||||
default:
|
||||
ui.req.Metadata[k] = v
|
||||
}
|
||||
@@ -4621,6 +4837,25 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
if o.fs.opt.StorageClass != "" {
|
||||
ui.req.StorageClass = types.StorageClass(o.fs.opt.StorageClass)
|
||||
}
|
||||
|
||||
// Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
|
||||
// "copy" means: keep the value from metadata (already applied above in the switch)
|
||||
if !o.fs.opt.ObjectLockSetAfterUpload {
|
||||
if o.fs.opt.ObjectLockMode != "" && !strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") {
|
||||
ui.req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(o.fs.opt.ObjectLockMode))
|
||||
}
|
||||
if o.fs.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") {
|
||||
retainDate, err := parseRetainUntilDate(o.fs.opt.ObjectLockRetainUntilDate)
|
||||
if err != nil {
|
||||
return ui, fmt.Errorf("invalid object_lock_retain_until_date %q: %w", o.fs.opt.ObjectLockRetainUntilDate, err)
|
||||
}
|
||||
ui.req.ObjectLockRetainUntilDate = &retainDate
|
||||
}
|
||||
if o.fs.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") {
|
||||
ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(o.fs.opt.ObjectLockLegalHoldStatus))
|
||||
}
|
||||
}
|
||||
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
@@ -4744,6 +4979,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
fs.Debugf(o, "Multipart upload Etag: %s OK", wantETag)
|
||||
}
|
||||
|
||||
// Set Object Lock via separate API calls if requested
|
||||
if o.fs.opt.ObjectLockSetAfterUpload {
|
||||
if err := o.setObjectLockAfterUpload(ctx, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4761,6 +5004,9 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
if o.fs.opt.RequesterPays {
|
||||
req.RequestPayer = types.RequestPayerRequester
|
||||
}
|
||||
if o.fs.opt.BypassGovernanceRetention {
|
||||
req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
|
||||
}
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := o.fs.c.DeleteObject(ctx, &req)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
@@ -4768,6 +5014,120 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// setObjectRetention sets Object Lock retention on an object via PutObjectRetention API
|
||||
//
|
||||
// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
|
||||
// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
|
||||
// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
|
||||
// still require Content-MD5 for PutObjectRetention requests.
|
||||
// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
|
||||
func (o *Object) setObjectRetention(ctx context.Context, mode types.ObjectLockRetentionMode, retainUntilDate time.Time) error {
|
||||
bucket, bucketPath := o.split()
|
||||
req := s3.PutObjectRetentionInput{
|
||||
Bucket: &bucket,
|
||||
Key: &bucketPath,
|
||||
VersionId: o.versionID,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: mode,
|
||||
RetainUntilDate: &retainUntilDate,
|
||||
},
|
||||
}
|
||||
if o.fs.opt.RequesterPays {
|
||||
req.RequestPayer = types.RequestPayerRequester
|
||||
}
|
||||
if o.fs.opt.BypassGovernanceRetention {
|
||||
req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
|
||||
}
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := o.fs.c.PutObjectRetention(ctx, &req,
|
||||
s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
|
||||
// setObjectLegalHold sets Object Lock legal hold on an object via PutObjectLegalHold API
|
||||
//
|
||||
// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
|
||||
// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
|
||||
// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
|
||||
// still require Content-MD5 for PutObjectLegalHold requests.
|
||||
// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
|
||||
func (o *Object) setObjectLegalHold(ctx context.Context, status types.ObjectLockLegalHoldStatus) error {
|
||||
bucket, bucketPath := o.split()
|
||||
req := s3.PutObjectLegalHoldInput{
|
||||
Bucket: &bucket,
|
||||
Key: &bucketPath,
|
||||
VersionId: o.versionID,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: status,
|
||||
},
|
||||
}
|
||||
if o.fs.opt.RequesterPays {
|
||||
req.RequestPayer = types.RequestPayerRequester
|
||||
}
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := o.fs.c.PutObjectLegalHold(ctx, &req,
|
||||
s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
|
||||
// setObjectLockAfterUpload sets Object Lock via separate API calls after upload
|
||||
// This is for S3 providers that don't support Object Lock headers during PUT
|
||||
func (o *Object) setObjectLockAfterUpload(ctx context.Context, src fs.ObjectInfo) error {
|
||||
// Determine the mode
|
||||
var mode types.ObjectLockRetentionMode
|
||||
modeOpt := o.fs.opt.ObjectLockMode
|
||||
if strings.EqualFold(modeOpt, "copy") {
|
||||
if srcObj, ok := src.(*Object); ok && srcObj.objectLockMode != nil {
|
||||
mode = types.ObjectLockRetentionMode(*srcObj.objectLockMode)
|
||||
}
|
||||
} else if modeOpt != "" {
|
||||
mode = types.ObjectLockRetentionMode(strings.ToUpper(modeOpt))
|
||||
}
|
||||
|
||||
// Determine the retain until date
|
||||
var retainUntilDate time.Time
|
||||
dateOpt := o.fs.opt.ObjectLockRetainUntilDate
|
||||
if strings.EqualFold(dateOpt, "copy") {
|
||||
if srcObj, ok := src.(*Object); ok && srcObj.objectLockRetainUntilDate != nil {
|
||||
retainUntilDate = *srcObj.objectLockRetainUntilDate
|
||||
}
|
||||
} else if dateOpt != "" {
|
||||
var err error
|
||||
retainUntilDate, err = parseRetainUntilDate(dateOpt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid object_lock_retain_until_date %q: %w", dateOpt, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set retention if both mode and date are set
|
||||
if mode != "" && !retainUntilDate.IsZero() {
|
||||
if err := o.setObjectRetention(ctx, mode, retainUntilDate); err != nil {
|
||||
return fmt.Errorf("failed to set object retention: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine and set legal hold
|
||||
var legalHold types.ObjectLockLegalHoldStatus
|
||||
legalHoldOpt := o.fs.opt.ObjectLockLegalHoldStatus
|
||||
if strings.EqualFold(legalHoldOpt, "copy") {
|
||||
if srcObj, ok := src.(*Object); ok && srcObj.objectLockLegalHoldStatus != nil {
|
||||
legalHold = types.ObjectLockLegalHoldStatus(*srcObj.objectLockLegalHoldStatus)
|
||||
}
|
||||
} else if legalHoldOpt != "" {
|
||||
legalHold = types.ObjectLockLegalHoldStatus(strings.ToUpper(legalHoldOpt))
|
||||
}
|
||||
|
||||
if legalHold != "" {
|
||||
if err := o.setObjectLegalHold(ctx, legalHold); err != nil {
|
||||
return fmt.Errorf("failed to set legal hold: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
err := o.readMetaData(ctx)
|
||||
@@ -4811,7 +5171,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata = make(fs.Metadata, len(o.meta)+7)
|
||||
metadata = make(fs.Metadata, len(o.meta)+10)
|
||||
for k, v := range o.meta {
|
||||
switch k {
|
||||
case metaMtime:
|
||||
@@ -4846,6 +5206,15 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
|
||||
setMetadata("content-disposition", o.contentDisposition)
|
||||
setMetadata("content-encoding", o.contentEncoding)
|
||||
setMetadata("content-language", o.contentLanguage)
|
||||
|
||||
// Set Object Lock metadata
|
||||
setMetadata("object-lock-mode", o.objectLockMode)
|
||||
if o.objectLockRetainUntilDate != nil {
|
||||
formatted := o.objectLockRetainUntilDate.Format(time.RFC3339)
|
||||
setMetadata("object-lock-retain-until-date", &formatted)
|
||||
}
|
||||
setMetadata("object-lock-legal-hold-status", o.objectLockLegalHoldStatus)
|
||||
|
||||
metadata["tier"] = o.GetTier()
|
||||
|
||||
return metadata, nil
|
||||
|
||||
@@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
// Purge gets tested later
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a temporary bucket with Object Lock enabled to test on.
|
||||
// This exercises our BucketObjectLockEnabled option and isolates
|
||||
// the test from the main test bucket.
|
||||
lockBucket := f.rootBucket + "-object-lock-" + random.String(8)
|
||||
lockBucket = strings.ToLower(lockBucket)
|
||||
|
||||
// Try to create bucket with Object Lock enabled
|
||||
objectLockEnabled := true
|
||||
req := s3.CreateBucketInput{
|
||||
Bucket: &lockBucket,
|
||||
ACL: types.BucketCannedACL(f.opt.BucketACL),
|
||||
ObjectLockEnabledForBucket: &objectLockEnabled,
|
||||
}
|
||||
if f.opt.LocationConstraint != "" {
|
||||
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
|
||||
LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
|
||||
}
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.CreateBucket(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify Object Lock is actually enabled on the new bucket.
|
||||
// Some S3-compatible servers (e.g. rclone serve s3) accept the
|
||||
// ObjectLockEnabledForBucket flag but don't actually implement Object Lock.
|
||||
var lockCfg *s3.GetObjectLockConfigurationOutput
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: &lockBucket,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil || lockCfg.ObjectLockConfiguration == nil ||
|
||||
lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err)
|
||||
}
|
||||
|
||||
// Switch f to use the Object Lock bucket for this test
|
||||
oldBucket := f.rootBucket
|
||||
oldRoot := f.root
|
||||
oldRootDir := f.rootDirectory
|
||||
f.rootBucket = lockBucket
|
||||
f.root = lockBucket
|
||||
f.rootDirectory = ""
|
||||
defer func() {
|
||||
f.rootBucket = oldBucket
|
||||
f.root = oldRoot
|
||||
f.rootDirectory = oldRootDir
|
||||
}()
|
||||
|
||||
// Helper to remove an object with Object Lock protection
|
||||
removeLocked := func(t *testing.T, obj fs.Object) {
|
||||
t.Helper()
|
||||
o := obj.(*Object)
|
||||
// Remove legal hold if present
|
||||
_ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
|
||||
// Enable bypass governance retention for deletion
|
||||
o.fs.opt.BypassGovernanceRetention = true
|
||||
err := obj.Remove(ctx)
|
||||
o.fs.opt.BypassGovernanceRetention = false
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Clean up the temporary bucket after all sub-tests
|
||||
defer func() {
|
||||
// List and remove all object versions
|
||||
var objectVersions []types.ObjectIdentifier
|
||||
listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket}
|
||||
for {
|
||||
var resp *s3.ListObjectVersionsOutput
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.c.ListObjectVersions(ctx, listReq)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Failed to list object versions for cleanup: %v", err)
|
||||
break
|
||||
}
|
||||
for _, v := range resp.Versions {
|
||||
objectVersions = append(objectVersions, types.ObjectIdentifier{
|
||||
Key: v.Key,
|
||||
VersionId: v.VersionId,
|
||||
})
|
||||
}
|
||||
for _, m := range resp.DeleteMarkers {
|
||||
objectVersions = append(objectVersions, types.ObjectIdentifier{
|
||||
Key: m.Key,
|
||||
VersionId: m.VersionId,
|
||||
})
|
||||
}
|
||||
if !aws.ToBool(resp.IsTruncated) {
|
||||
break
|
||||
}
|
||||
listReq.KeyMarker = resp.NextKeyMarker
|
||||
listReq.VersionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
if len(objectVersions) > 0 {
|
||||
bypass := true
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &lockBucket,
|
||||
BypassGovernanceRetention: &bypass,
|
||||
Delete: &types.Delete{
|
||||
Objects: objectVersions,
|
||||
Quiet: aws.Bool(true),
|
||||
},
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
}()
|
||||
|
||||
retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second)
|
||||
|
||||
t.Run("Retention", func(t *testing.T) {
|
||||
// Set Object Lock options for this test
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
defer func() {
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
}()
|
||||
|
||||
// Upload an object with Object Lock retention
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Read back metadata and verify Object Lock settings
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
|
||||
})
|
||||
|
||||
t.Run("LegalHold", func(t *testing.T) {
|
||||
// Set Object Lock legal hold option
|
||||
f.opt.ObjectLockLegalHoldStatus = "ON"
|
||||
defer func() {
|
||||
f.opt.ObjectLockLegalHoldStatus = ""
|
||||
}()
|
||||
|
||||
// Upload an object with legal hold
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Verify legal hold is ON
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
|
||||
|
||||
// Set legal hold to OFF
|
||||
err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Clear cached metadata and re-read
|
||||
o.meta = nil
|
||||
gotMetadata, err = o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"])
|
||||
})
|
||||
|
||||
t.Run("SetAfterUpload", func(t *testing.T) {
|
||||
// Test the post-upload API path (PutObjectRetention + PutObjectLegalHold)
|
||||
f.opt.ObjectLockSetAfterUpload = true
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
f.opt.ObjectLockLegalHoldStatus = "ON"
|
||||
defer func() {
|
||||
f.opt.ObjectLockSetAfterUpload = false
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
f.opt.ObjectLockLegalHoldStatus = ""
|
||||
}()
|
||||
|
||||
// Upload an object - lock applied AFTER upload via separate API calls
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Verify all Object Lock settings were applied
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
|
||||
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
t.Run("NoHead", f.InternalTestNoHead)
|
||||
t.Run("Versions", f.InternalTestVersions)
|
||||
t.Run("ObjectLock", f.InternalTestObjectLock)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -92,3 +93,87 @@ var (
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
_ fstests.SetCopyCutoffer = (*Fs)(nil)
|
||||
)
|
||||
|
||||
func TestParseRetainUntilDate(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
checkFunc func(t *testing.T, result time.Time)
|
||||
}{
|
||||
{
|
||||
name: "RFC3339 date",
|
||||
input: "2030-01-15T10:30:00Z",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z")
|
||||
assert.Equal(t, expected, result)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "RFC3339 date with timezone",
|
||||
input: "2030-06-15T10:30:00+02:00",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00")
|
||||
assert.Equal(t, expected, result)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration days",
|
||||
input: "365d",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(365 * 24 * time.Hour)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration hours",
|
||||
input: "24h",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(24 * time.Hour)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration minutes",
|
||||
input: "30m",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(30 * time.Minute)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid input",
|
||||
input: "not-a-date",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := parseRetainUntilDate(tt.input)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
if tt.checkFunc != nil {
|
||||
tt.checkFunc(t, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -904,6 +904,48 @@ section, small files that are not uploaded as multipart, use a different tag, ca
|
||||
the upload to fail. A simple solution is to set the `--s3-upload-cutoff 0` and force
|
||||
all the files to be uploaded as multipart.
|
||||
|
||||
#### Setting Object Lock retention
|
||||
|
||||
Rclone supports setting Object Lock retention on uploaded objects with
|
||||
the following options:
|
||||
|
||||
- `--s3-object-lock-mode` - Set the Object Lock mode (GOVERNANCE or COMPLIANCE)
|
||||
- `--s3-object-lock-retain-until-date` - Set the retention date (RFC3339 format or duration like `365d`, `24h`)
|
||||
- `--s3-object-lock-legal-hold-status` - Set legal hold (ON or OFF)
|
||||
|
||||
Example - upload files with 1 year GOVERNANCE retention:
|
||||
|
||||
rclone copy local:/data remote:bucket \
|
||||
--s3-object-lock-mode GOVERNANCE \
|
||||
--s3-object-lock-retain-until-date 365d
|
||||
|
||||
#### Copying Object Lock settings from source
|
||||
|
||||
When using `--metadata`, you can use the special value `copy` to preserve
|
||||
the source object's Object Lock settings:
|
||||
|
||||
rclone copy source:bucket dest:bucket \
|
||||
--metadata \
|
||||
--s3-object-lock-mode copy \
|
||||
--s3-object-lock-retain-until-date copy
|
||||
|
||||
You can also mix copied and explicit values. For example, to change the
|
||||
mode from COMPLIANCE to GOVERNANCE while preserving the original retention date:
|
||||
|
||||
rclone copy source:bucket dest:bucket \
|
||||
--metadata \
|
||||
--s3-object-lock-mode GOVERNANCE \
|
||||
--s3-object-lock-retain-until-date copy
|
||||
|
||||
#### Additional Object Lock options
|
||||
|
||||
- `--s3-bypass-governance-retention` - Required to delete or overwrite objects
|
||||
with GOVERNANCE mode retention before the retention date expires
|
||||
- `--s3-bucket-object-lock-enabled` - Enable Object Lock when creating a new
|
||||
bucket with `rclone mkdir`
|
||||
- `--s3-object-lock-set-after-upload` - Set Object Lock via separate API calls
|
||||
after upload (for providers that don't support Object Lock headers during PUT)
|
||||
|
||||
<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
|
||||
### Standard options
|
||||
|
||||
|
||||
Reference in New Issue
Block a user