diff --git a/README.md b/README.md index f9536e85d..4ab45416d 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ directories to and from different cloud storage providers. - Dropbox [:page_facing_up:](https://rclone.org/dropbox/) - Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/) - Exaba [:page_facing_up:](https://rclone.org/s3/#exaba) +- Fastly Object Storage [:page_facing_up:](https://rclone.org/s3/#fastly) - Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files) - FileLu [:page_facing_up:](https://rclone.org/filelu/) - Filen [:page_facing_up:](https://rclone.org/filen/) diff --git a/backend/s3/README.md b/backend/s3/README.md index 014218b09..d0090ffb3 100644 --- a/backend/s3/README.md +++ b/backend/s3/README.md @@ -170,6 +170,7 @@ In `backend/s3/provider/YourProvider.yaml` UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` UseXID *bool `yaml:"use_x_id,omitempty"` SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` + EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"` CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"` MaxUploadParts *int `yaml:"max_upload_parts,omitempty"` MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"` diff --git a/backend/s3/provider/Fastly.yaml b/backend/s3/provider/Fastly.yaml new file mode 100644 index 000000000..8ec71c744 --- /dev/null +++ b/backend/s3/provider/Fastly.yaml @@ -0,0 +1,16 @@ +name: Fastly +description: Fastly Object Storage +region: + us-east: US East + us-west: US West + eu-central: EU Central +endpoint: + us-east.object.fastlystorage.app: US East + us-west.object.fastlystorage.app: US West + eu-central.object.fastlystorage.app: EU Central +quirks: + force_path_style: true + use_already_exists: false + use_multipart_etag: false + use_multipart_uploads: false + etag_is_not_md5: true diff --git a/backend/s3/providers.go b/backend/s3/providers.go index 858aaea6e..1a7500ea3 100644 --- a/backend/s3/providers.go +++ b/backend/s3/providers.go @@ -32,6 +32,7 @@ type Quirks struct { UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` UseXID *bool `yaml:"use_x_id,omitempty"` SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` + EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"` CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"` MaxUploadParts *int `yaml:"max_upload_parts,omitempty"` MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"` diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 246077e3b..b5bba8327 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1664,6 +1664,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // MD5 digest of their object data. f.etagIsNotMD5 = true } + if provider.Quirks.EtagIsNotMD5 != nil && *provider.Quirks.EtagIsNotMD5 { + // Provider always returns ETags that are not MD5 (e.g., mandatory encryption) + f.etagIsNotMD5 = true + } if opt.DirectoryBucket { // Objects uploaded to directory buckets appear to have random ETags // @@ -1688,6 +1692,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.Provider == "AWS" { f.features.DoubleSlash = true } + if opt.Provider == "Fastly" { + f.features.Copy = nil + } if opt.Provider == "Rabata" { f.features.Copy = nil } diff --git a/docs/content/_index.md b/docs/content/_index.md index 6c9f26182..1ded74b1b 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -133,6 +133,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}} {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}} {{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}} +{{< provider name="Fastly Object Storage" home="https://www.fastly.com/products/storage" config="/s3/#fastly" >}} {{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}} {{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}} {{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index 96d6664ce..9a4a23d04 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -23,6 +23,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}} +{{< provider name="Fastly Object Storage" home="https://www.fastly.com/products/storage" config="/s3/#fastly" >}} {{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}} {{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}} {{< provider name="Hetzner" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}} @@ -5279,6 +5280,92 @@ secret_access_key = XXX endpoint = http://127.0.0.1:9000/ ``` +### Fastly Object Storage {#fastly} + +[Fastly Object Storage](https://www.fastly.com/products/storage) is an +S3-compatible object storage service from Fastly. It provides three +regions (US East, US West, and EU Central) with mandatory server-side +encryption. + +Here is an example of making a configuration. First run: + +```console +rclone config +``` + +This will guide you through an interactive setup process. + +```text +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter name for new remote. +name> fastly + +Option Storage. +Type of storage to configure. +Storage> s3 + +Option provider. +Choose your S3 provider. +provider> Fastly + +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +env_auth> false + +Option access_key_id. +AWS Access Key ID. +access_key_id> YOUR_ACCESS_KEY + +Option secret_access_key. +AWS Secret Access Key (password). +secret_access_key> YOUR_SECRET_KEY + +Option region. +Region where your bucket will be created and your data stored. +region> us-east + +Option endpoint. +Endpoint for S3 API. +endpoint> us-east.object.fastlystorage.app + +Edit advanced config? +y) Yes +n) No (default) +y/n> n + +Configuration complete. +Options: +- type: s3 +- provider: Fastly +- access_key_id: YOUR_ACCESS_KEY +- secret_access_key: YOUR_SECRET_KEY +- region: us-east +- endpoint: us-east.object.fastlystorage.app +Keep this "fastly" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +The resulting configuration file should look like: + +```ini +[fastly] +type = s3 +provider = Fastly +access_key_id = YOUR_ACCESS_KEY +secret_access_key = YOUR_SECRET_KEY +region = us-east +endpoint = us-east.object.fastlystorage.app +``` + ### Google Cloud Storage [GoogleCloudStorage](https://cloud.google.com/storage/docs) is an