1
0
mirror of https://github.com/rclone/rclone.git synced 2026-02-20 19:33:28 +00:00

s3: add Object Lock support

Add support for S3 Object Lock with the following new options:

- --s3-object-lock-mode: set retention mode (GOVERNANCE/COMPLIANCE/copy)
- --s3-object-lock-retain-until-date: set retention date (RFC3339/duration/copy)
- --s3-object-lock-legal-hold-status: set legal hold (ON/OFF/copy)
- --s3-bypass-governance-retention: bypass GOVERNANCE lock on delete
- --s3-bucket-object-lock-enabled: enable Object Lock on bucket creation
- --s3-object-lock-set-after-upload: apply lock via separate API calls

The special value "copy" preserves the source object's setting when used
with --metadata flag, enabling scenarios like cloning objects from
COMPLIANCE to GOVERNANCE mode while preserving the original retention date.

Includes integration tests that create a temporary Object Lock bucket covering:
- Retention Mode and Date
- Legal Hold
- Apply settings after upload
- Override protections using bypass-governance flag
The tests are gracefully skipped on providers that do not support Object Lock.

Fixes #4683
Closes #7894 #7893 #8866
This commit is contained in:
Chris
2026-02-20 17:40:24 +01:00
committed by GitHub
parent fd8b28d36d
commit 7d0a8bf850
4 changed files with 727 additions and 5 deletions

View File

@@ -828,6 +828,107 @@ use |-vv| to see the debug level logs.
}, {
Name: "ibm_resource_instance_id",
Help: "IBM service instance id",
}, {
Name: "object_lock_mode",
Help: `Object Lock mode to apply when uploading or copying objects.
Set this to apply Object Lock retention mode to objects.
If not set, no Object Lock mode is applied (even with --metadata).
Note: To enable Object Lock retention, you must set BOTH object_lock_mode
AND object_lock_retain_until_date. Setting only one has no effect.
- GOVERNANCE: Set Object Lock mode to GOVERNANCE
- COMPLIANCE: Set Object Lock mode to COMPLIANCE
- copy: Copy the mode from the source object (requires --metadata)
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "GOVERNANCE",
Help: "Set Object Lock mode to GOVERNANCE",
}, {
Value: "COMPLIANCE",
Help: "Set Object Lock mode to COMPLIANCE",
}, {
Value: "copy",
Help: "Copy from source object (requires --metadata)",
}},
}, {
Name: "object_lock_retain_until_date",
Help: `Object Lock retention until date to apply when uploading or copying objects.
Set this to apply Object Lock retention date to objects.
If not set, no retention date is applied (even with --metadata).
Note: To enable Object Lock retention, you must set BOTH object_lock_mode
AND object_lock_retain_until_date. Setting only one has no effect.
Accepts:
- RFC 3339 format: 2030-01-02T15:04:05Z
- Duration from now: 365d, 1y, 6M (days, years, months)
- copy: Copy the date from the source object (requires --metadata)
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "copy",
Help: "Copy from source object (requires --metadata)",
}, {
Value: "2030-01-01T00:00:00Z",
Help: "Set specific date (RFC 3339 format)",
}, {
Value: "365d",
Help: "Set retention for 365 days from now",
}, {
Value: "1y",
Help: "Set retention for 1 year from now",
}},
}, {
Name: "object_lock_legal_hold_status",
Help: `Object Lock legal hold status to apply when uploading or copying objects.
Set this to apply Object Lock legal hold to objects.
If not set, no legal hold is applied (even with --metadata).
Note: Legal hold is independent of retention and can be set separately.
- ON: Enable legal hold
- OFF: Disable legal hold
- copy: Copy the legal hold status from the source object (requires --metadata)
See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "ON",
Help: "Enable legal hold",
}, {
Value: "OFF",
Help: "Disable legal hold",
}, {
Value: "copy",
Help: "Copy from source object (requires --metadata)",
}},
}, {
Name: "bypass_governance_retention",
Help: `Allow deleting or modifying objects locked with GOVERNANCE mode.`,
Default: false,
Advanced: true,
}, {
Name: "bucket_object_lock_enabled",
Help: `Enable Object Lock when creating new buckets.`,
Default: false,
}, {
Name: "object_lock_set_after_upload",
Help: `Set Object Lock via separate API calls after upload.
Use this for S3-compatible providers that don't support setting Object Lock
headers during PUT operations. When enabled, Object Lock is set via separate
PutObjectRetention and PutObjectLegalHold API calls after the upload completes.
This adds extra API calls per object, so only enable if your provider requires it.`,
Default: false,
Advanced: true,
},
}}))
}
@@ -922,6 +1023,21 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
Example: "2006-01-02T15:04:05.999999999Z07:00",
ReadOnly: true,
},
"object-lock-mode": {
Help: "Object Lock mode: GOVERNANCE or COMPLIANCE",
Type: "string",
Example: "GOVERNANCE",
},
"object-lock-retain-until-date": {
Help: "Object Lock retention until date",
Type: "RFC 3339",
Example: "2030-01-02T15:04:05Z",
},
"object-lock-legal-hold-status": {
Help: "Object Lock legal hold status: ON or OFF",
Type: "string",
Example: "OFF",
},
}
// Options defines the configuration for this backend
@@ -992,6 +1108,12 @@ type Options struct {
IBMInstanceID string `config:"ibm_resource_instance_id"`
UseXID fs.Tristate `config:"use_x_id"`
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
ObjectLockMode string `config:"object_lock_mode"`
ObjectLockRetainUntilDate string `config:"object_lock_retain_until_date"`
ObjectLockLegalHoldStatus string `config:"object_lock_legal_hold_status"`
BypassGovernanceRetention bool `config:"bypass_governance_retention"`
BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"`
ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"`
}
// Fs represents a remote s3 server
@@ -1036,6 +1158,11 @@ type Object struct {
contentDisposition *string // Content-Disposition: header
contentEncoding *string // Content-Encoding: header
contentLanguage *string // Content-Language: header
// Object Lock metadata
objectLockMode *string // Object Lock mode: GOVERNANCE or COMPLIANCE
objectLockRetainUntilDate *time.Time // Object Lock retention until date
objectLockLegalHoldStatus *string // Object Lock legal hold: ON or OFF
}
// safely dereference the pointer, returning a zero T if nil
@@ -1056,6 +1183,21 @@ func getHTTPStatusCode(err error) int {
return -1
}
// parseRetainUntilDate parses a retain until date from a string.
// It accepts RFC 3339 format or duration strings like "365d", "1y", "6m".
func parseRetainUntilDate(s string) (time.Time, error) {
// First try RFC 3339 format
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t, nil
}
// Try as a duration from now
d, err := fs.ParseDuration(s)
if err != nil {
return time.Time{}, fmt.Errorf("can't parse %q as RFC 3339 date or duration: %w", s, err)
}
return time.Now().Add(d), nil
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
@@ -2649,8 +2791,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
}
return f.cache.Create(bucket, func() error {
req := s3.CreateBucketInput{
Bucket: &bucket,
ACL: types.BucketCannedACL(f.opt.BucketACL),
Bucket: &bucket,
ACL: types.BucketCannedACL(f.opt.BucketACL),
ObjectLockEnabledForBucket: &f.opt.BucketObjectLockEnabled,
}
if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
@@ -2769,6 +2912,24 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
req.StorageClass = types.StorageClass(f.opt.StorageClass)
}
// Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
// "copy" means: keep the value from source (passed via req from prepareUpload/setFrom functions)
if !f.opt.ObjectLockSetAfterUpload {
if f.opt.ObjectLockMode != "" && !strings.EqualFold(f.opt.ObjectLockMode, "copy") {
req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(f.opt.ObjectLockMode))
}
if f.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(f.opt.ObjectLockRetainUntilDate, "copy") {
retainDate, err := parseRetainUntilDate(f.opt.ObjectLockRetainUntilDate)
if err != nil {
return fmt.Errorf("invalid object_lock_retain_until_date: %w", err)
}
req.ObjectLockRetainUntilDate = &retainDate
}
if f.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(f.opt.ObjectLockLegalHoldStatus, "copy") {
req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(f.opt.ObjectLockLegalHoldStatus))
}
}
if src.bytes >= int64(f.opt.CopyCutoff) {
return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)
}
@@ -2951,7 +3112,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
}
setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
if ci.Metadata {
// Use REPLACE directive if metadata is being modified, otherwise S3 ignores our values
// This is needed when:
// 1. --metadata flag is set
// 2. Any Object Lock option is set (to override or explicitly copy)
needsReplace := ci.Metadata ||
f.opt.ObjectLockMode != "" ||
f.opt.ObjectLockRetainUntilDate != "" ||
f.opt.ObjectLockLegalHoldStatus != ""
if needsReplace {
req.MetadataDirective = types.MetadataDirectiveReplace
}
@@ -2959,7 +3128,21 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil {
return nil, err
}
return f.NewObject(ctx, remote)
dstObj, err := f.NewObject(ctx, remote)
if err != nil {
return nil, err
}
// Set Object Lock via separate API calls if requested
if f.opt.ObjectLockSetAfterUpload {
if dstObject, ok := dstObj.(*Object); ok {
if err := dstObject.setObjectLockAfterUpload(ctx, srcObj); err != nil {
return nil, err
}
}
}
return dstObj, nil
}
// Hashes returns the supported hash sets.
@@ -3838,6 +4021,19 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
// Set Object Lock metadata
if resp.ObjectLockMode != "" {
mode := string(resp.ObjectLockMode)
o.objectLockMode = &mode
}
if resp.ObjectLockRetainUntilDate != nil {
o.objectLockRetainUntilDate = resp.ObjectLockRetainUntilDate
}
if resp.ObjectLockLegalHoldStatus != "" {
status := string(resp.ObjectLockLegalHoldStatus)
o.objectLockLegalHoldStatus = &status
}
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" {
o.bytes = -1
@@ -4556,6 +4752,26 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
case "btime":
// write as metadata since we can't set it
ui.req.Metadata[k] = v
case "object-lock-mode":
// Only apply if option is set to "copy" and not using after-upload API
if strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
ui.req.ObjectLockMode = types.ObjectLockMode(v)
}
case "object-lock-retain-until-date":
// Only apply if option is set to "copy" and not using after-upload API
if strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
retainDate, err := time.Parse(time.RFC3339, v)
if err != nil {
fs.Debugf(o, "failed to parse object-lock-retain-until-date %q: %v", v, err)
} else {
ui.req.ObjectLockRetainUntilDate = &retainDate
}
}
case "object-lock-legal-hold-status":
// Only apply if option is set to "copy" and not using after-upload API
if strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(v)
}
default:
ui.req.Metadata[k] = v
}
@@ -4621,6 +4837,25 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
if o.fs.opt.StorageClass != "" {
ui.req.StorageClass = types.StorageClass(o.fs.opt.StorageClass)
}
// Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
// "copy" means: keep the value from metadata (already applied above in the switch)
if !o.fs.opt.ObjectLockSetAfterUpload {
if o.fs.opt.ObjectLockMode != "" && !strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") {
ui.req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(o.fs.opt.ObjectLockMode))
}
if o.fs.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") {
retainDate, err := parseRetainUntilDate(o.fs.opt.ObjectLockRetainUntilDate)
if err != nil {
return ui, fmt.Errorf("invalid object_lock_retain_until_date %q: %w", o.fs.opt.ObjectLockRetainUntilDate, err)
}
ui.req.ObjectLockRetainUntilDate = &retainDate
}
if o.fs.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") {
ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(o.fs.opt.ObjectLockLegalHoldStatus))
}
}
// Apply upload options
for _, option := range options {
key, value := option.Header()
@@ -4744,6 +4979,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
fs.Debugf(o, "Multipart upload Etag: %s OK", wantETag)
}
// Set Object Lock via separate API calls if requested
if o.fs.opt.ObjectLockSetAfterUpload {
if err := o.setObjectLockAfterUpload(ctx, src); err != nil {
return err
}
}
return err
}
@@ -4761,6 +5004,9 @@ func (o *Object) Remove(ctx context.Context) error {
if o.fs.opt.RequesterPays {
req.RequestPayer = types.RequestPayerRequester
}
if o.fs.opt.BypassGovernanceRetention {
req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
}
err := o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.DeleteObject(ctx, &req)
return o.fs.shouldRetry(ctx, err)
@@ -4768,6 +5014,120 @@ func (o *Object) Remove(ctx context.Context) error {
return err
}
// setObjectRetention sets Object Lock retention on an object via PutObjectRetention API
//
// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
// still require Content-MD5 for PutObjectRetention requests.
// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
func (o *Object) setObjectRetention(ctx context.Context, mode types.ObjectLockRetentionMode, retainUntilDate time.Time) error {
bucket, bucketPath := o.split()
req := s3.PutObjectRetentionInput{
Bucket: &bucket,
Key: &bucketPath,
VersionId: o.versionID,
Retention: &types.ObjectLockRetention{
Mode: mode,
RetainUntilDate: &retainUntilDate,
},
}
if o.fs.opt.RequesterPays {
req.RequestPayer = types.RequestPayerRequester
}
if o.fs.opt.BypassGovernanceRetention {
req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
}
return o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.PutObjectRetention(ctx, &req,
s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
return o.fs.shouldRetry(ctx, err)
})
}
// setObjectLegalHold sets Object Lock legal hold on an object via PutObjectLegalHold API
//
// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
// still require Content-MD5 for PutObjectLegalHold requests.
// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
func (o *Object) setObjectLegalHold(ctx context.Context, status types.ObjectLockLegalHoldStatus) error {
bucket, bucketPath := o.split()
req := s3.PutObjectLegalHoldInput{
Bucket: &bucket,
Key: &bucketPath,
VersionId: o.versionID,
LegalHold: &types.ObjectLockLegalHold{
Status: status,
},
}
if o.fs.opt.RequesterPays {
req.RequestPayer = types.RequestPayerRequester
}
return o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.PutObjectLegalHold(ctx, &req,
s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
return o.fs.shouldRetry(ctx, err)
})
}
// setObjectLockAfterUpload sets Object Lock via separate API calls after upload
// This is for S3 providers that don't support Object Lock headers during PUT
func (o *Object) setObjectLockAfterUpload(ctx context.Context, src fs.ObjectInfo) error {
// Determine the mode
var mode types.ObjectLockRetentionMode
modeOpt := o.fs.opt.ObjectLockMode
if strings.EqualFold(modeOpt, "copy") {
if srcObj, ok := src.(*Object); ok && srcObj.objectLockMode != nil {
mode = types.ObjectLockRetentionMode(*srcObj.objectLockMode)
}
} else if modeOpt != "" {
mode = types.ObjectLockRetentionMode(strings.ToUpper(modeOpt))
}
// Determine the retain until date
var retainUntilDate time.Time
dateOpt := o.fs.opt.ObjectLockRetainUntilDate
if strings.EqualFold(dateOpt, "copy") {
if srcObj, ok := src.(*Object); ok && srcObj.objectLockRetainUntilDate != nil {
retainUntilDate = *srcObj.objectLockRetainUntilDate
}
} else if dateOpt != "" {
var err error
retainUntilDate, err = parseRetainUntilDate(dateOpt)
if err != nil {
return fmt.Errorf("invalid object_lock_retain_until_date %q: %w", dateOpt, err)
}
}
// Set retention if both mode and date are set
if mode != "" && !retainUntilDate.IsZero() {
if err := o.setObjectRetention(ctx, mode, retainUntilDate); err != nil {
return fmt.Errorf("failed to set object retention: %w", err)
}
}
// Determine and set legal hold
var legalHold types.ObjectLockLegalHoldStatus
legalHoldOpt := o.fs.opt.ObjectLockLegalHoldStatus
if strings.EqualFold(legalHoldOpt, "copy") {
if srcObj, ok := src.(*Object); ok && srcObj.objectLockLegalHoldStatus != nil {
legalHold = types.ObjectLockLegalHoldStatus(*srcObj.objectLockLegalHoldStatus)
}
} else if legalHoldOpt != "" {
legalHold = types.ObjectLockLegalHoldStatus(strings.ToUpper(legalHoldOpt))
}
if legalHold != "" {
if err := o.setObjectLegalHold(ctx, legalHold); err != nil {
return fmt.Errorf("failed to set legal hold: %w", err)
}
}
return nil
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType(ctx context.Context) string {
err := o.readMetaData(ctx)
@@ -4811,7 +5171,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
if err != nil {
return nil, err
}
metadata = make(fs.Metadata, len(o.meta)+7)
metadata = make(fs.Metadata, len(o.meta)+10)
for k, v := range o.meta {
switch k {
case metaMtime:
@@ -4846,6 +5206,15 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
setMetadata("content-disposition", o.contentDisposition)
setMetadata("content-encoding", o.contentEncoding)
setMetadata("content-language", o.contentLanguage)
// Set Object Lock metadata
setMetadata("object-lock-mode", o.objectLockMode)
if o.objectLockRetainUntilDate != nil {
formatted := o.objectLockRetainUntilDate.Format(time.RFC3339)
setMetadata("object-lock-retain-until-date", &formatted)
}
setMetadata("object-lock-legal-hold-status", o.objectLockLegalHoldStatus)
metadata["tier"] = o.GetTier()
return metadata, nil

View File

@@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
// Purge gets tested later
}
func (f *Fs) InternalTestObjectLock(t *testing.T) {
ctx := context.Background()
// Create a temporary bucket with Object Lock enabled to test on.
// This exercises our BucketObjectLockEnabled option and isolates
// the test from the main test bucket.
lockBucket := f.rootBucket + "-object-lock-" + random.String(8)
lockBucket = strings.ToLower(lockBucket)
// Try to create bucket with Object Lock enabled
objectLockEnabled := true
req := s3.CreateBucketInput{
Bucket: &lockBucket,
ACL: types.BucketCannedACL(f.opt.BucketACL),
ObjectLockEnabledForBucket: &objectLockEnabled,
}
if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
}
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.c.CreateBucket(ctx, &req)
return f.shouldRetry(ctx, err)
})
if err != nil {
t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err)
}
// Verify Object Lock is actually enabled on the new bucket.
// Some S3-compatible servers (e.g. rclone serve s3) accept the
// ObjectLockEnabledForBucket flag but don't actually implement Object Lock.
var lockCfg *s3.GetObjectLockConfigurationOutput
err = f.pacer.Call(func() (bool, error) {
var err error
lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
Bucket: &lockBucket,
})
return f.shouldRetry(ctx, err)
})
if err != nil || lockCfg.ObjectLockConfiguration == nil ||
lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
_ = f.pacer.Call(func() (bool, error) {
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
return f.shouldRetry(ctx, err)
})
t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err)
}
// Switch f to use the Object Lock bucket for this test
oldBucket := f.rootBucket
oldRoot := f.root
oldRootDir := f.rootDirectory
f.rootBucket = lockBucket
f.root = lockBucket
f.rootDirectory = ""
defer func() {
f.rootBucket = oldBucket
f.root = oldRoot
f.rootDirectory = oldRootDir
}()
// Helper to remove an object with Object Lock protection
removeLocked := func(t *testing.T, obj fs.Object) {
t.Helper()
o := obj.(*Object)
// Remove legal hold if present
_ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
// Enable bypass governance retention for deletion
o.fs.opt.BypassGovernanceRetention = true
err := obj.Remove(ctx)
o.fs.opt.BypassGovernanceRetention = false
assert.NoError(t, err)
}
// Clean up the temporary bucket after all sub-tests
defer func() {
// List and remove all object versions
var objectVersions []types.ObjectIdentifier
listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket}
for {
var resp *s3.ListObjectVersionsOutput
err := f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.c.ListObjectVersions(ctx, listReq)
return f.shouldRetry(ctx, err)
})
if err != nil {
t.Logf("Failed to list object versions for cleanup: %v", err)
break
}
for _, v := range resp.Versions {
objectVersions = append(objectVersions, types.ObjectIdentifier{
Key: v.Key,
VersionId: v.VersionId,
})
}
for _, m := range resp.DeleteMarkers {
objectVersions = append(objectVersions, types.ObjectIdentifier{
Key: m.Key,
VersionId: m.VersionId,
})
}
if !aws.ToBool(resp.IsTruncated) {
break
}
listReq.KeyMarker = resp.NextKeyMarker
listReq.VersionIdMarker = resp.NextVersionIdMarker
}
if len(objectVersions) > 0 {
bypass := true
_ = f.pacer.Call(func() (bool, error) {
_, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: &lockBucket,
BypassGovernanceRetention: &bypass,
Delete: &types.Delete{
Objects: objectVersions,
Quiet: aws.Bool(true),
},
})
return f.shouldRetry(ctx, err)
})
}
_ = f.pacer.Call(func() (bool, error) {
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
return f.shouldRetry(ctx, err)
})
}()
retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second)
t.Run("Retention", func(t *testing.T) {
// Set Object Lock options for this test
f.opt.ObjectLockMode = "GOVERNANCE"
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
defer func() {
f.opt.ObjectLockMode = ""
f.opt.ObjectLockRetainUntilDate = ""
}()
// Upload an object with Object Lock retention
contents := random.String(100)
item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
defer func() {
removeLocked(t, obj)
}()
// Read back metadata and verify Object Lock settings
o := obj.(*Object)
gotMetadata, err := o.Metadata(ctx)
require.NoError(t, err)
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
require.NoError(t, err)
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
})
t.Run("LegalHold", func(t *testing.T) {
// Set Object Lock legal hold option
f.opt.ObjectLockLegalHoldStatus = "ON"
defer func() {
f.opt.ObjectLockLegalHoldStatus = ""
}()
// Upload an object with legal hold
contents := random.String(100)
item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
defer func() {
removeLocked(t, obj)
}()
// Verify legal hold is ON
o := obj.(*Object)
gotMetadata, err := o.Metadata(ctx)
require.NoError(t, err)
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
// Set legal hold to OFF
err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
require.NoError(t, err)
// Clear cached metadata and re-read
o.meta = nil
gotMetadata, err = o.Metadata(ctx)
require.NoError(t, err)
assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"])
})
t.Run("SetAfterUpload", func(t *testing.T) {
// Test the post-upload API path (PutObjectRetention + PutObjectLegalHold)
f.opt.ObjectLockSetAfterUpload = true
f.opt.ObjectLockMode = "GOVERNANCE"
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
f.opt.ObjectLockLegalHoldStatus = "ON"
defer func() {
f.opt.ObjectLockSetAfterUpload = false
f.opt.ObjectLockMode = ""
f.opt.ObjectLockRetainUntilDate = ""
f.opt.ObjectLockLegalHoldStatus = ""
}()
// Upload an object - lock applied AFTER upload via separate API calls
contents := random.String(100)
item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
defer func() {
removeLocked(t, obj)
}()
// Verify all Object Lock settings were applied
o := obj.(*Object)
gotMetadata, err := o.Metadata(ctx)
require.NoError(t, err)
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
require.NoError(t, err)
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
})
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Metadata", f.InternalTestMetadata)
t.Run("NoHead", f.InternalTestNoHead)
t.Run("Versions", f.InternalTestVersions)
t.Run("ObjectLock", f.InternalTestObjectLock)
}
var _ fstests.InternalTester = (*Fs)(nil)

View File

@@ -5,6 +5,7 @@ import (
"context"
"net/http"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/rclone/rclone/fs"
@@ -92,3 +93,87 @@ var (
_ fstests.SetUploadCutoffer = (*Fs)(nil)
_ fstests.SetCopyCutoffer = (*Fs)(nil)
)
func TestParseRetainUntilDate(t *testing.T) {
now := time.Now()
tests := []struct {
name string
input string
wantErr bool
checkFunc func(t *testing.T, result time.Time)
}{
{
name: "RFC3339 date",
input: "2030-01-15T10:30:00Z",
wantErr: false,
checkFunc: func(t *testing.T, result time.Time) {
expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z")
assert.Equal(t, expected, result)
},
},
{
name: "RFC3339 date with timezone",
input: "2030-06-15T10:30:00+02:00",
wantErr: false,
checkFunc: func(t *testing.T, result time.Time) {
expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00")
assert.Equal(t, expected, result)
},
},
{
name: "duration days",
input: "365d",
wantErr: false,
checkFunc: func(t *testing.T, result time.Time) {
expected := now.Add(365 * 24 * time.Hour)
diff := result.Sub(expected)
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now")
},
},
{
name: "duration hours",
input: "24h",
wantErr: false,
checkFunc: func(t *testing.T, result time.Time) {
expected := now.Add(24 * time.Hour)
diff := result.Sub(expected)
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now")
},
},
{
name: "duration minutes",
input: "30m",
wantErr: false,
checkFunc: func(t *testing.T, result time.Time) {
expected := now.Add(30 * time.Minute)
diff := result.Sub(expected)
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now")
},
},
{
name: "invalid input",
input: "not-a-date",
wantErr: true,
},
{
name: "empty input",
input: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := parseRetainUntilDate(tt.input)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
if tt.checkFunc != nil {
tt.checkFunc(t, result)
}
})
}
}

View File

@@ -904,6 +904,48 @@ section, small files that are not uploaded as multipart, use a different tag, ca
the upload to fail. A simple solution is to set the `--s3-upload-cutoff 0` and force
all the files to be uploaded as multipart.
#### Setting Object Lock retention
Rclone supports setting Object Lock retention on uploaded objects with
the following options:
- `--s3-object-lock-mode` - Set the Object Lock mode (GOVERNANCE or COMPLIANCE)
- `--s3-object-lock-retain-until-date` - Set the retention date (RFC3339 format or duration like `365d`, `24h`)
- `--s3-object-lock-legal-hold-status` - Set legal hold (ON or OFF)
Example - upload files with 1 year GOVERNANCE retention:
rclone copy local:/data remote:bucket \
--s3-object-lock-mode GOVERNANCE \
--s3-object-lock-retain-until-date 365d
#### Copying Object Lock settings from source
When using `--metadata`, you can use the special value `copy` to preserve
the source object's Object Lock settings:
rclone copy source:bucket dest:bucket \
--metadata \
--s3-object-lock-mode copy \
--s3-object-lock-retain-until-date copy
You can also mix copied and explicit values. For example, to change the
mode from COMPLIANCE to GOVERNANCE while preserving the original retention date:
rclone copy source:bucket dest:bucket \
--metadata \
--s3-object-lock-mode GOVERNANCE \
--s3-object-lock-retain-until-date copy
#### Additional Object Lock options
- `--s3-bypass-governance-retention` - Required to delete or overwrite objects
with GOVERNANCE mode retention before the retention date expires
- `--s3-bucket-object-lock-enabled` - Enable Object Lock when creating a new
bucket with `rclone mkdir`
- `--s3-object-lock-set-after-upload` - Set Object Lock via separate API calls
after upload (for providers that don't support Object Lock headers during PUT)
<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
### Standard options