From 7d0a8bf850b93d3f70a50f47eab489295e6058d9 Mon Sep 17 00:00:00 2001 From: Chris <238498929+chris081519-crypto@users.noreply.github.com> Date: Fri, 20 Feb 2026 17:40:24 +0100 Subject: [PATCH] s3: add Object Lock support Add support for S3 Object Lock with the following new options: - --s3-object-lock-mode: set retention mode (GOVERNANCE/COMPLIANCE/copy) - --s3-object-lock-retain-until-date: set retention date (RFC3339/duration/copy) - --s3-object-lock-legal-hold-status: set legal hold (ON/OFF/copy) - --s3-bypass-governance-retention: bypass GOVERNANCE lock on delete - --s3-bucket-object-lock-enabled: enable Object Lock on bucket creation - --s3-object-lock-set-after-upload: apply lock via separate API calls The special value "copy" preserves the source object's setting when used with --metadata flag, enabling scenarios like cloning objects from COMPLIANCE to GOVERNANCE mode while preserving the original retention date. Includes integration tests that create a temporary Object Lock bucket covering: - Retention Mode and Date - Legal Hold - Apply settings after upload - Override protections using bypass-governance flag The tests are gracefully skipped on providers that do not support Object Lock. Fixes #4683 Closes #7894 #7893 #8866 --- backend/s3/s3.go | 379 ++++++++++++++++++++++++++++++++- backend/s3/s3_internal_test.go | 226 ++++++++++++++++++++ backend/s3/s3_test.go | 85 ++++++++ docs/content/s3.md | 42 ++++ 4 files changed, 727 insertions(+), 5 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index b5bba8327..02398a34f 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -828,6 +828,107 @@ use |-vv| to see the debug level logs. }, { Name: "ibm_resource_instance_id", Help: "IBM service instance id", + }, { + Name: "object_lock_mode", + Help: `Object Lock mode to apply when uploading or copying objects. + +Set this to apply Object Lock retention mode to objects. +If not set, no Object Lock mode is applied (even with --metadata). + +Note: To enable Object Lock retention, you must set BOTH object_lock_mode +AND object_lock_retain_until_date. Setting only one has no effect. + +- GOVERNANCE: Set Object Lock mode to GOVERNANCE +- COMPLIANCE: Set Object Lock mode to COMPLIANCE +- copy: Copy the mode from the source object (requires --metadata) + +See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "GOVERNANCE", + Help: "Set Object Lock mode to GOVERNANCE", + }, { + Value: "COMPLIANCE", + Help: "Set Object Lock mode to COMPLIANCE", + }, { + Value: "copy", + Help: "Copy from source object (requires --metadata)", + }}, + }, { + Name: "object_lock_retain_until_date", + Help: `Object Lock retention until date to apply when uploading or copying objects. + +Set this to apply Object Lock retention date to objects. +If not set, no retention date is applied (even with --metadata). + +Note: To enable Object Lock retention, you must set BOTH object_lock_mode +AND object_lock_retain_until_date. Setting only one has no effect. + +Accepts: +- RFC 3339 format: 2030-01-02T15:04:05Z +- Duration from now: 365d, 1y, 6M (days, years, months) +- copy: Copy the date from the source object (requires --metadata) + +See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "copy", + Help: "Copy from source object (requires --metadata)", + }, { + Value: "2030-01-01T00:00:00Z", + Help: "Set specific date (RFC 3339 format)", + }, { + Value: "365d", + Help: "Set retention for 365 days from now", + }, { + Value: "1y", + Help: "Set retention for 1 year from now", + }}, + }, { + Name: "object_lock_legal_hold_status", + Help: `Object Lock legal hold status to apply when uploading or copying objects. + +Set this to apply Object Lock legal hold to objects. +If not set, no legal hold is applied (even with --metadata). + +Note: Legal hold is independent of retention and can be set separately. + +- ON: Enable legal hold +- OFF: Disable legal hold +- copy: Copy the legal hold status from the source object (requires --metadata) + +See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`, + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "ON", + Help: "Enable legal hold", + }, { + Value: "OFF", + Help: "Disable legal hold", + }, { + Value: "copy", + Help: "Copy from source object (requires --metadata)", + }}, + }, { + Name: "bypass_governance_retention", + Help: `Allow deleting or modifying objects locked with GOVERNANCE mode.`, + Default: false, + Advanced: true, + }, { + Name: "bucket_object_lock_enabled", + Help: `Enable Object Lock when creating new buckets.`, + Default: false, + }, { + Name: "object_lock_set_after_upload", + Help: `Set Object Lock via separate API calls after upload. + +Use this for S3-compatible providers that don't support setting Object Lock +headers during PUT operations. When enabled, Object Lock is set via separate +PutObjectRetention and PutObjectLegalHold API calls after the upload completes. + +This adds extra API calls per object, so only enable if your provider requires it.`, + Default: false, + Advanced: true, }, }})) } @@ -922,6 +1023,21 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{ Example: "2006-01-02T15:04:05.999999999Z07:00", ReadOnly: true, }, + "object-lock-mode": { + Help: "Object Lock mode: GOVERNANCE or COMPLIANCE", + Type: "string", + Example: "GOVERNANCE", + }, + "object-lock-retain-until-date": { + Help: "Object Lock retention until date", + Type: "RFC 3339", + Example: "2030-01-02T15:04:05Z", + }, + "object-lock-legal-hold-status": { + Help: "Object Lock legal hold status: ON or OFF", + Type: "string", + Example: "OFF", + }, } // Options defines the configuration for this backend @@ -992,6 +1108,12 @@ type Options struct { IBMInstanceID string `config:"ibm_resource_instance_id"` UseXID fs.Tristate `config:"use_x_id"` SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"` + ObjectLockMode string `config:"object_lock_mode"` + ObjectLockRetainUntilDate string `config:"object_lock_retain_until_date"` + ObjectLockLegalHoldStatus string `config:"object_lock_legal_hold_status"` + BypassGovernanceRetention bool `config:"bypass_governance_retention"` + BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"` + ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"` } // Fs represents a remote s3 server @@ -1036,6 +1158,11 @@ type Object struct { contentDisposition *string // Content-Disposition: header contentEncoding *string // Content-Encoding: header contentLanguage *string // Content-Language: header + + // Object Lock metadata + objectLockMode *string // Object Lock mode: GOVERNANCE or COMPLIANCE + objectLockRetainUntilDate *time.Time // Object Lock retention until date + objectLockLegalHoldStatus *string // Object Lock legal hold: ON or OFF } // safely dereference the pointer, returning a zero T if nil @@ -1056,6 +1183,21 @@ func getHTTPStatusCode(err error) int { return -1 } +// parseRetainUntilDate parses a retain until date from a string. +// It accepts RFC 3339 format or duration strings like "365d", "1y", "6m". +func parseRetainUntilDate(s string) (time.Time, error) { + // First try RFC 3339 format + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t, nil + } + // Try as a duration from now + d, err := fs.ParseDuration(s) + if err != nil { + return time.Time{}, fmt.Errorf("can't parse %q as RFC 3339 date or duration: %w", s, err) + } + return time.Now().Add(d), nil +} + // ------------------------------------------------------------ // Name of the remote (as passed into NewFs) @@ -2649,8 +2791,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error { } return f.cache.Create(bucket, func() error { req := s3.CreateBucketInput{ - Bucket: &bucket, - ACL: types.BucketCannedACL(f.opt.BucketACL), + Bucket: &bucket, + ACL: types.BucketCannedACL(f.opt.BucketACL), + ObjectLockEnabledForBucket: &f.opt.BucketObjectLockEnabled, } if f.opt.LocationConstraint != "" { req.CreateBucketConfiguration = &types.CreateBucketConfiguration{ @@ -2769,6 +2912,24 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa req.StorageClass = types.StorageClass(f.opt.StorageClass) } + // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set) + // "copy" means: keep the value from source (passed via req from prepareUpload/setFrom functions) + if !f.opt.ObjectLockSetAfterUpload { + if f.opt.ObjectLockMode != "" && !strings.EqualFold(f.opt.ObjectLockMode, "copy") { + req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(f.opt.ObjectLockMode)) + } + if f.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(f.opt.ObjectLockRetainUntilDate, "copy") { + retainDate, err := parseRetainUntilDate(f.opt.ObjectLockRetainUntilDate) + if err != nil { + return fmt.Errorf("invalid object_lock_retain_until_date: %w", err) + } + req.ObjectLockRetainUntilDate = &retainDate + } + if f.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(f.opt.ObjectLockLegalHoldStatus, "copy") { + req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(f.opt.ObjectLockLegalHoldStatus)) + } + } + if src.bytes >= int64(f.opt.CopyCutoff) { return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src) } @@ -2951,7 +3112,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req) - if ci.Metadata { + // Use REPLACE directive if metadata is being modified, otherwise S3 ignores our values + // This is needed when: + // 1. --metadata flag is set + // 2. Any Object Lock option is set (to override or explicitly copy) + needsReplace := ci.Metadata || + f.opt.ObjectLockMode != "" || + f.opt.ObjectLockRetainUntilDate != "" || + f.opt.ObjectLockLegalHoldStatus != "" + if needsReplace { req.MetadataDirective = types.MetadataDirectiveReplace } @@ -2959,7 +3128,21 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, if err != nil { return nil, err } - return f.NewObject(ctx, remote) + dstObj, err := f.NewObject(ctx, remote) + if err != nil { + return nil, err + } + + // Set Object Lock via separate API calls if requested + if f.opt.ObjectLockSetAfterUpload { + if dstObject, ok := dstObj.(*Object); ok { + if err := dstObject.setObjectLockAfterUpload(ctx, srcObj); err != nil { + return nil, err + } + } + } + + return dstObj, nil } // Hashes returns the supported hash sets. @@ -3838,6 +4021,19 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) { o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding)) o.contentLanguage = stringClonePointer(resp.ContentLanguage) + // Set Object Lock metadata + if resp.ObjectLockMode != "" { + mode := string(resp.ObjectLockMode) + o.objectLockMode = &mode + } + if resp.ObjectLockRetainUntilDate != nil { + o.objectLockRetainUntilDate = resp.ObjectLockRetainUntilDate + } + if resp.ObjectLockLegalHoldStatus != "" { + status := string(resp.ObjectLockLegalHoldStatus) + o.objectLockLegalHoldStatus = &status + } + // If decompressing then size and md5sum are unknown if o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" { o.bytes = -1 @@ -4556,6 +4752,26 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [ case "btime": // write as metadata since we can't set it ui.req.Metadata[k] = v + case "object-lock-mode": + // Only apply if option is set to "copy" and not using after-upload API + if strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") && !o.fs.opt.ObjectLockSetAfterUpload { + ui.req.ObjectLockMode = types.ObjectLockMode(v) + } + case "object-lock-retain-until-date": + // Only apply if option is set to "copy" and not using after-upload API + if strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") && !o.fs.opt.ObjectLockSetAfterUpload { + retainDate, err := time.Parse(time.RFC3339, v) + if err != nil { + fs.Debugf(o, "failed to parse object-lock-retain-until-date %q: %v", v, err) + } else { + ui.req.ObjectLockRetainUntilDate = &retainDate + } + } + case "object-lock-legal-hold-status": + // Only apply if option is set to "copy" and not using after-upload API + if strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") && !o.fs.opt.ObjectLockSetAfterUpload { + ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(v) + } default: ui.req.Metadata[k] = v } @@ -4621,6 +4837,25 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [ if o.fs.opt.StorageClass != "" { ui.req.StorageClass = types.StorageClass(o.fs.opt.StorageClass) } + + // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set) + // "copy" means: keep the value from metadata (already applied above in the switch) + if !o.fs.opt.ObjectLockSetAfterUpload { + if o.fs.opt.ObjectLockMode != "" && !strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") { + ui.req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(o.fs.opt.ObjectLockMode)) + } + if o.fs.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") { + retainDate, err := parseRetainUntilDate(o.fs.opt.ObjectLockRetainUntilDate) + if err != nil { + return ui, fmt.Errorf("invalid object_lock_retain_until_date %q: %w", o.fs.opt.ObjectLockRetainUntilDate, err) + } + ui.req.ObjectLockRetainUntilDate = &retainDate + } + if o.fs.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") { + ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(o.fs.opt.ObjectLockLegalHoldStatus)) + } + } + // Apply upload options for _, option := range options { key, value := option.Header() @@ -4744,6 +4979,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } fs.Debugf(o, "Multipart upload Etag: %s OK", wantETag) } + + // Set Object Lock via separate API calls if requested + if o.fs.opt.ObjectLockSetAfterUpload { + if err := o.setObjectLockAfterUpload(ctx, src); err != nil { + return err + } + } + return err } @@ -4761,6 +5004,9 @@ func (o *Object) Remove(ctx context.Context) error { if o.fs.opt.RequesterPays { req.RequestPayer = types.RequestPayerRequester } + if o.fs.opt.BypassGovernanceRetention { + req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention + } err := o.fs.pacer.Call(func() (bool, error) { _, err := o.fs.c.DeleteObject(ctx, &req) return o.fs.shouldRetry(ctx, err) @@ -4768,6 +5014,120 @@ func (o *Object) Remove(ctx context.Context) error { return err } +// setObjectRetention sets Object Lock retention on an object via PutObjectRetention API +// +// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is +// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as +// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO) +// still require Content-MD5 for PutObjectRetention requests. +// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960 +func (o *Object) setObjectRetention(ctx context.Context, mode types.ObjectLockRetentionMode, retainUntilDate time.Time) error { + bucket, bucketPath := o.split() + req := s3.PutObjectRetentionInput{ + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, + Retention: &types.ObjectLockRetention{ + Mode: mode, + RetainUntilDate: &retainUntilDate, + }, + } + if o.fs.opt.RequesterPays { + req.RequestPayer = types.RequestPayerRequester + } + if o.fs.opt.BypassGovernanceRetention { + req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention + } + return o.fs.pacer.Call(func() (bool, error) { + _, err := o.fs.c.PutObjectRetention(ctx, &req, + s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware)) + return o.fs.shouldRetry(ctx, err) + }) +} + +// setObjectLegalHold sets Object Lock legal hold on an object via PutObjectLegalHold API +// +// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is +// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as +// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO) +// still require Content-MD5 for PutObjectLegalHold requests. +// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960 +func (o *Object) setObjectLegalHold(ctx context.Context, status types.ObjectLockLegalHoldStatus) error { + bucket, bucketPath := o.split() + req := s3.PutObjectLegalHoldInput{ + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, + LegalHold: &types.ObjectLockLegalHold{ + Status: status, + }, + } + if o.fs.opt.RequesterPays { + req.RequestPayer = types.RequestPayerRequester + } + return o.fs.pacer.Call(func() (bool, error) { + _, err := o.fs.c.PutObjectLegalHold(ctx, &req, + s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware)) + return o.fs.shouldRetry(ctx, err) + }) +} + +// setObjectLockAfterUpload sets Object Lock via separate API calls after upload +// This is for S3 providers that don't support Object Lock headers during PUT +func (o *Object) setObjectLockAfterUpload(ctx context.Context, src fs.ObjectInfo) error { + // Determine the mode + var mode types.ObjectLockRetentionMode + modeOpt := o.fs.opt.ObjectLockMode + if strings.EqualFold(modeOpt, "copy") { + if srcObj, ok := src.(*Object); ok && srcObj.objectLockMode != nil { + mode = types.ObjectLockRetentionMode(*srcObj.objectLockMode) + } + } else if modeOpt != "" { + mode = types.ObjectLockRetentionMode(strings.ToUpper(modeOpt)) + } + + // Determine the retain until date + var retainUntilDate time.Time + dateOpt := o.fs.opt.ObjectLockRetainUntilDate + if strings.EqualFold(dateOpt, "copy") { + if srcObj, ok := src.(*Object); ok && srcObj.objectLockRetainUntilDate != nil { + retainUntilDate = *srcObj.objectLockRetainUntilDate + } + } else if dateOpt != "" { + var err error + retainUntilDate, err = parseRetainUntilDate(dateOpt) + if err != nil { + return fmt.Errorf("invalid object_lock_retain_until_date %q: %w", dateOpt, err) + } + } + + // Set retention if both mode and date are set + if mode != "" && !retainUntilDate.IsZero() { + if err := o.setObjectRetention(ctx, mode, retainUntilDate); err != nil { + return fmt.Errorf("failed to set object retention: %w", err) + } + } + + // Determine and set legal hold + var legalHold types.ObjectLockLegalHoldStatus + legalHoldOpt := o.fs.opt.ObjectLockLegalHoldStatus + if strings.EqualFold(legalHoldOpt, "copy") { + if srcObj, ok := src.(*Object); ok && srcObj.objectLockLegalHoldStatus != nil { + legalHold = types.ObjectLockLegalHoldStatus(*srcObj.objectLockLegalHoldStatus) + } + } else if legalHoldOpt != "" { + legalHold = types.ObjectLockLegalHoldStatus(strings.ToUpper(legalHoldOpt)) + } + + if legalHold != "" { + if err := o.setObjectLegalHold(ctx, legalHold); err != nil { + return fmt.Errorf("failed to set legal hold: %w", err) + } + } + + return nil +} + // MimeType of an Object if known, "" otherwise func (o *Object) MimeType(ctx context.Context) string { err := o.readMetaData(ctx) @@ -4811,7 +5171,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) if err != nil { return nil, err } - metadata = make(fs.Metadata, len(o.meta)+7) + metadata = make(fs.Metadata, len(o.meta)+10) for k, v := range o.meta { switch k { case metaMtime: @@ -4846,6 +5206,15 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) setMetadata("content-disposition", o.contentDisposition) setMetadata("content-encoding", o.contentEncoding) setMetadata("content-language", o.contentLanguage) + + // Set Object Lock metadata + setMetadata("object-lock-mode", o.objectLockMode) + if o.objectLockRetainUntilDate != nil { + formatted := o.objectLockRetainUntilDate.Format(time.RFC3339) + setMetadata("object-lock-retain-until-date", &formatted) + } + setMetadata("object-lock-legal-hold-status", o.objectLockLegalHoldStatus) + metadata["tier"] = o.GetTier() return metadata, nil diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index cdca649ae..c4e6637f5 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) { // Purge gets tested later } +func (f *Fs) InternalTestObjectLock(t *testing.T) { + ctx := context.Background() + + // Create a temporary bucket with Object Lock enabled to test on. + // This exercises our BucketObjectLockEnabled option and isolates + // the test from the main test bucket. + lockBucket := f.rootBucket + "-object-lock-" + random.String(8) + lockBucket = strings.ToLower(lockBucket) + + // Try to create bucket with Object Lock enabled + objectLockEnabled := true + req := s3.CreateBucketInput{ + Bucket: &lockBucket, + ACL: types.BucketCannedACL(f.opt.BucketACL), + ObjectLockEnabledForBucket: &objectLockEnabled, + } + if f.opt.LocationConstraint != "" { + req.CreateBucketConfiguration = &types.CreateBucketConfiguration{ + LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint), + } + } + err := f.pacer.Call(func() (bool, error) { + _, err := f.c.CreateBucket(ctx, &req) + return f.shouldRetry(ctx, err) + }) + if err != nil { + t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err) + } + + // Verify Object Lock is actually enabled on the new bucket. + // Some S3-compatible servers (e.g. rclone serve s3) accept the + // ObjectLockEnabledForBucket flag but don't actually implement Object Lock. + var lockCfg *s3.GetObjectLockConfigurationOutput + err = f.pacer.Call(func() (bool, error) { + var err error + lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{ + Bucket: &lockBucket, + }) + return f.shouldRetry(ctx, err) + }) + if err != nil || lockCfg.ObjectLockConfiguration == nil || + lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled { + _ = f.pacer.Call(func() (bool, error) { + _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket}) + return f.shouldRetry(ctx, err) + }) + t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err) + } + + // Switch f to use the Object Lock bucket for this test + oldBucket := f.rootBucket + oldRoot := f.root + oldRootDir := f.rootDirectory + f.rootBucket = lockBucket + f.root = lockBucket + f.rootDirectory = "" + defer func() { + f.rootBucket = oldBucket + f.root = oldRoot + f.rootDirectory = oldRootDir + }() + + // Helper to remove an object with Object Lock protection + removeLocked := func(t *testing.T, obj fs.Object) { + t.Helper() + o := obj.(*Object) + // Remove legal hold if present + _ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff) + // Enable bypass governance retention for deletion + o.fs.opt.BypassGovernanceRetention = true + err := obj.Remove(ctx) + o.fs.opt.BypassGovernanceRetention = false + assert.NoError(t, err) + } + + // Clean up the temporary bucket after all sub-tests + defer func() { + // List and remove all object versions + var objectVersions []types.ObjectIdentifier + listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket} + for { + var resp *s3.ListObjectVersionsOutput + err := f.pacer.Call(func() (bool, error) { + var err error + resp, err = f.c.ListObjectVersions(ctx, listReq) + return f.shouldRetry(ctx, err) + }) + if err != nil { + t.Logf("Failed to list object versions for cleanup: %v", err) + break + } + for _, v := range resp.Versions { + objectVersions = append(objectVersions, types.ObjectIdentifier{ + Key: v.Key, + VersionId: v.VersionId, + }) + } + for _, m := range resp.DeleteMarkers { + objectVersions = append(objectVersions, types.ObjectIdentifier{ + Key: m.Key, + VersionId: m.VersionId, + }) + } + if !aws.ToBool(resp.IsTruncated) { + break + } + listReq.KeyMarker = resp.NextKeyMarker + listReq.VersionIdMarker = resp.NextVersionIdMarker + } + if len(objectVersions) > 0 { + bypass := true + _ = f.pacer.Call(func() (bool, error) { + _, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &lockBucket, + BypassGovernanceRetention: &bypass, + Delete: &types.Delete{ + Objects: objectVersions, + Quiet: aws.Bool(true), + }, + }) + return f.shouldRetry(ctx, err) + }) + } + _ = f.pacer.Call(func() (bool, error) { + _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket}) + return f.shouldRetry(ctx, err) + }) + }() + + retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second) + + t.Run("Retention", func(t *testing.T) { + // Set Object Lock options for this test + f.opt.ObjectLockMode = "GOVERNANCE" + f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339) + defer func() { + f.opt.ObjectLockMode = "" + f.opt.ObjectLockRetainUntilDate = "" + }() + + // Upload an object with Object Lock retention + contents := random.String(100) + item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + removeLocked(t, obj) + }() + + // Read back metadata and verify Object Lock settings + o := obj.(*Object) + gotMetadata, err := o.Metadata(ctx) + require.NoError(t, err) + + assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"]) + gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"]) + require.NoError(t, err) + assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second) + }) + + t.Run("LegalHold", func(t *testing.T) { + // Set Object Lock legal hold option + f.opt.ObjectLockLegalHoldStatus = "ON" + defer func() { + f.opt.ObjectLockLegalHoldStatus = "" + }() + + // Upload an object with legal hold + contents := random.String(100) + item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + removeLocked(t, obj) + }() + + // Verify legal hold is ON + o := obj.(*Object) + gotMetadata, err := o.Metadata(ctx) + require.NoError(t, err) + assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"]) + + // Set legal hold to OFF + err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff) + require.NoError(t, err) + + // Clear cached metadata and re-read + o.meta = nil + gotMetadata, err = o.Metadata(ctx) + require.NoError(t, err) + assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"]) + }) + + t.Run("SetAfterUpload", func(t *testing.T) { + // Test the post-upload API path (PutObjectRetention + PutObjectLegalHold) + f.opt.ObjectLockSetAfterUpload = true + f.opt.ObjectLockMode = "GOVERNANCE" + f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339) + f.opt.ObjectLockLegalHoldStatus = "ON" + defer func() { + f.opt.ObjectLockSetAfterUpload = false + f.opt.ObjectLockMode = "" + f.opt.ObjectLockRetainUntilDate = "" + f.opt.ObjectLockLegalHoldStatus = "" + }() + + // Upload an object - lock applied AFTER upload via separate API calls + contents := random.String(100) + item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + removeLocked(t, obj) + }() + + // Verify all Object Lock settings were applied + o := obj.(*Object) + gotMetadata, err := o.Metadata(ctx) + require.NoError(t, err) + + assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"]) + gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"]) + require.NoError(t, err) + assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second) + assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"]) + }) +} + func (f *Fs) InternalTest(t *testing.T) { t.Run("Metadata", f.InternalTestMetadata) t.Run("NoHead", f.InternalTestNoHead) t.Run("Versions", f.InternalTestVersions) + t.Run("ObjectLock", f.InternalTestObjectLock) } var _ fstests.InternalTester = (*Fs)(nil) diff --git a/backend/s3/s3_test.go b/backend/s3/s3_test.go index 973314c5c..a262d1703 100644 --- a/backend/s3/s3_test.go +++ b/backend/s3/s3_test.go @@ -5,6 +5,7 @@ import ( "context" "net/http" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/rclone/rclone/fs" @@ -92,3 +93,87 @@ var ( _ fstests.SetUploadCutoffer = (*Fs)(nil) _ fstests.SetCopyCutoffer = (*Fs)(nil) ) + +func TestParseRetainUntilDate(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + input string + wantErr bool + checkFunc func(t *testing.T, result time.Time) + }{ + { + name: "RFC3339 date", + input: "2030-01-15T10:30:00Z", + wantErr: false, + checkFunc: func(t *testing.T, result time.Time) { + expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z") + assert.Equal(t, expected, result) + }, + }, + { + name: "RFC3339 date with timezone", + input: "2030-06-15T10:30:00+02:00", + wantErr: false, + checkFunc: func(t *testing.T, result time.Time) { + expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00") + assert.Equal(t, expected, result) + }, + }, + { + name: "duration days", + input: "365d", + wantErr: false, + checkFunc: func(t *testing.T, result time.Time) { + expected := now.Add(365 * 24 * time.Hour) + diff := result.Sub(expected) + assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now") + }, + }, + { + name: "duration hours", + input: "24h", + wantErr: false, + checkFunc: func(t *testing.T, result time.Time) { + expected := now.Add(24 * time.Hour) + diff := result.Sub(expected) + assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now") + }, + }, + { + name: "duration minutes", + input: "30m", + wantErr: false, + checkFunc: func(t *testing.T, result time.Time) { + expected := now.Add(30 * time.Minute) + diff := result.Sub(expected) + assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now") + }, + }, + { + name: "invalid input", + input: "not-a-date", + wantErr: true, + }, + { + name: "empty input", + input: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := parseRetainUntilDate(tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + if tt.checkFunc != nil { + tt.checkFunc(t, result) + } + }) + } +} diff --git a/docs/content/s3.md b/docs/content/s3.md index b04a735eb..a368a2a8d 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -904,6 +904,48 @@ section, small files that are not uploaded as multipart, use a different tag, ca the upload to fail. A simple solution is to set the `--s3-upload-cutoff 0` and force all the files to be uploaded as multipart. +#### Setting Object Lock retention + +Rclone supports setting Object Lock retention on uploaded objects with +the following options: + +- `--s3-object-lock-mode` - Set the Object Lock mode (GOVERNANCE or COMPLIANCE) +- `--s3-object-lock-retain-until-date` - Set the retention date (RFC3339 format or duration like `365d`, `24h`) +- `--s3-object-lock-legal-hold-status` - Set legal hold (ON or OFF) + +Example - upload files with 1 year GOVERNANCE retention: + + rclone copy local:/data remote:bucket \ + --s3-object-lock-mode GOVERNANCE \ + --s3-object-lock-retain-until-date 365d + +#### Copying Object Lock settings from source + +When using `--metadata`, you can use the special value `copy` to preserve +the source object's Object Lock settings: + + rclone copy source:bucket dest:bucket \ + --metadata \ + --s3-object-lock-mode copy \ + --s3-object-lock-retain-until-date copy + +You can also mix copied and explicit values. For example, to change the +mode from COMPLIANCE to GOVERNANCE while preserving the original retention date: + + rclone copy source:bucket dest:bucket \ + --metadata \ + --s3-object-lock-mode GOVERNANCE \ + --s3-object-lock-retain-until-date copy + +#### Additional Object Lock options + +- `--s3-bypass-governance-retention` - Required to delete or overwrite objects + with GOVERNANCE mode retention before the retention date expires +- `--s3-bucket-object-lock-enabled` - Enable Object Lock when creating a new + bucket with `rclone mkdir` +- `--s3-object-lock-set-after-upload` - Set Object Lock via separate API calls + after upload (for providers that don't support Object Lock headers during PUT) + ### Standard options