fs: Debug the names of env vars we read values from as part of the config

See: https://forum.rclone.org/t/how-can-i-check-if-environment-variables-are-being-used/17490

Makefile

@@ -61,6 +61,10 @@ vars:
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
+btest:
+	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
+	@echo "Copied markdown of beta release to clip board"
+
 version:
 	@echo '$(TAG)'
 

backend/crypt/crypt.go

@@ -72,6 +72,20 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different crypt configs.
+
+Normally this option is not what you want, but if you have two crypts
+pointing to the same backend you can use it.
+
+This can be used, for example, to change file name encryption type
+without re-uploading all the data. Just make two crypt backends
+pointing to two different directories with the single changed
+parameter and use rclone move to move the files between the crypt
+remotes.`,
+			Advanced: true,
 		}, {
 			Name: "show_mapping",
 			Help: `For all files listed show how the names encrypt.
@@ -181,6 +195,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		CanHaveEmptyDirectories: true,
 		SetTier:                 true,
 		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -193,6 +208,7 @@ type Options struct {
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
+	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
 }
 

backend/googlecloudstorage/googlecloudstorage.go

@@ -298,7 +298,6 @@ type Object struct {
 	bytes    int64     // Bytes in the object
 	modTime  time.Time // Modified time of the object
 	mimeType string
-	gzipped  bool // set if object has Content-Encoding: gzip
 }
 
 // ------------------------------------------------------------
@@ -900,7 +899,6 @@ func (o *Object) setMetaData(info *storage.Object) {
 	o.url = info.MediaLink
 	o.bytes = int64(info.Size)
 	o.mimeType = info.ContentType
-	o.gzipped = info.ContentEncoding == "gzip"
 
 	// Read md5sum
 	md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@@ -1028,15 +1026,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	fs.FixRangeOption(options, o.bytes)
-	if o.gzipped {
-		// Allow files which are stored on the cloud storage system
-		// compressed to be downloaded without being decompressed.  Note
-		// that setting this here overrides the automatic decompression
-		// in the Transport.
-		//
-		// See: https://cloud.google.com/storage/docs/transcoding
-		req.Header.Set("Accept-Encoding", "gzip")
-	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/local/local.go

@@ -90,7 +90,24 @@ are being uploaded and aborts with a message which starts "can't copy
 
 However on some file systems this modification time check may fail (eg
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
-check can be disabled with this flag.`,
+check can be disabled with this flag.
+
+If this flag is set, rclone will use its best efforts to transfer a
+file which is being updated. If the file is only having things
+appended to it (eg a log) then rclone will transfer the log file with
+the size it had the first time rclone saw it.
+
+If the file is being modified throughout (not just appended to) then
+the transfer may fail with a hash check failure.
+
+In detail, once the file has had stat() called on it for the first
+time we:
+
+- Only transfer the size that stat gave
+- Only checksum the size that stat gave
+- Don't update the stat info for the file
+
+`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -809,6 +826,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		} else {
 			in, err = o.openTranslatedLink(0, -1)
 		}
+		// If not checking for updates, only read size given
+		if o.fs.opt.NoCheckUpdated {
+			in = readers.NewLimitedReadCloser(in, o.size)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -955,6 +976,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 	}
 
+	// If not checking updated then limit to current size.  This means if
+	// file is being extended, readers will read a o.Size() bytes rather
+	// than the new size making for a consistent upload.
+	if limit < 0 && o.fs.opt.NoCheckUpdated {
+		limit = o.size
+	}
+
 	// Handle a translated link
 	if o.translatedLink {
 		return o.openTranslatedLink(offset, limit)
@@ -1151,6 +1179,10 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
+	// if not checking updated then don't update the stat
+	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
+		return
+	}
 	// Don't overwrite the info if we don't need to
 	// this avoids upsetting the race detector
 	if o.size != info.Size() {

backend/s3/s3.go

@@ -40,6 +40,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -1204,7 +1205,9 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
-		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint).
+		WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
+
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}

fs/config/config.go

@@ -172,7 +172,7 @@ func makeConfigPath() string {
 	// Check to see if user supplied a --config variable or environment
 	// variable.  We can't use pflag for this because it isn't initialised
 	// yet so we search the command line manually.
-	_, configSupplied := os.LookupEnv("RCLONE_CONFIG")
+	_, configSupplied := fs.LookupEnv("RCLONE_CONFIG")
 	if !configSupplied {
 		for _, item := range os.Args {
 			if item == "--config" || strings.HasPrefix(item, "--config=") {
@@ -311,7 +311,7 @@ func loadConfigFile() (*goconfig.ConfigFile, error) {
 		} else {
 			usingPasswordCommand = false
 
-			envpw := os.Getenv("RCLONE_CONFIG_PASS")
+			envpw := fs.Getenv("RCLONE_CONFIG_PASS")
 
 			if envpw != "" {
 				err := setConfigPassword(envpw)
@@ -1432,7 +1432,7 @@ func FileGetFlag(section, key string) (string, bool) {
 // It looks up defaults in the environment if they are present
 func FileGet(section, key string, defaultVal ...string) string {
 	envKey := fs.ConfigToEnv(section, key)
-	newValue, found := os.LookupEnv(envKey)
+	newValue, found := fs.LookupEnv(envKey)
 	if found {
 		defaultVal = []string{newValue}
 	}

fs/config/flags/flags.go

@@ -4,7 +4,6 @@ package flags
 
 import (
 	"log"
-	"os"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -15,7 +14,7 @@ import (
 // sets the default from the environment if possible.
 func setDefaultFromEnv(flags *pflag.FlagSet, name string) {
 	key := fs.OptionToEnv(name)
-	newValue, found := os.LookupEnv(key)
+	newValue, found := fs.LookupEnv(key)
 	if found {
 		flag := flags.Lookup(name)
 		if flag == nil {

fs/fs.go

@@ -1193,9 +1193,25 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err e
 // A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name
 type configEnvVars string
 
+// LookupEnv calls os.LookupEnv and Debugfs a message if the env var was found
+func LookupEnv(key string) (value string, ok bool) {
+	value, ok = os.LookupEnv(key)
+	if ok {
+		Debugf(nil, "Read env var %s", key)
+	}
+	return value, ok
+}
+
+// Getenv calls os.LookupEnv and Debugfs a message if the env var was
+// found. If the var wasn't round it returns empty string.
+func Getenv(key string) (value string) {
+	value, _ = LookupEnv(key)
+	return value
+}
+
 // Get a config item from the environment variables if possible
 func (configName configEnvVars) Get(key string) (value string, ok bool) {
-	return os.LookupEnv(ConfigToEnv(string(configName), key))
+	return LookupEnv(ConfigToEnv(string(configName), key))
 }
 
 // A configmap.Getter to read from the environment RCLONE_option_name
@@ -1211,12 +1227,12 @@ func (oev optionEnvVars) Get(key string) (value string, ok bool) {
 	}
 	// For options with NoPrefix set, check without prefix too
 	if opt.NoPrefix {
-		value, ok = os.LookupEnv(OptionToEnv(key))
+		value, ok = LookupEnv(OptionToEnv(key))
 		if ok {
 			return value, ok
 		}
 	}
-	return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
+	return LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
 }
 
 // A configmap.Getter to read either the default value or the set