config create: add --non-interactive and --continue parameters #3455

This adds a mechanism to add external interfaces to rclone using the
state based configuration.

fs/backend_config.go

@@ -258,10 +258,42 @@ func StatePop(state string) (newState string, value string) {
 //
 // It wraps any OAuth transactions as necessary so only straight forward config questions are emitted
 func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
+	for {
+		out, err = backendConfigStep(ctx, name, m, ri, in)
+		if err != nil {
+			break
+		}
+		if out == nil || out.State == "" {
+			// finished
+			break
+		}
+		if out.Option != nil {
+			// question to ask user
+			break
+		}
+		if out.Error != "" {
+			// error to show user
+			break
+		}
+		// non terminal state, but no question to ask or error to show - loop here
+		in = ConfigIn{
+			State:  out.State,
+			Result: out.Result,
+		}
+	}
+	return out, err
+}
+
+func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, in ConfigIn) (out *ConfigOut, err error) {
 	ci := GetConfig(ctx)
 	if ri.Config == nil {
 		return nil, nil
 	}
+	Debugf(name, "config in: state=%q, result=%q", in.State, in.Result)
+	defer func() {
+		Debugf(name, "config out: out=%+v, err=%v", out, err)
+	}()
+
 	switch {
 	case strings.HasPrefix(in.State, "*oauth"):
 		// Do internal oauth states
@@ -299,7 +331,7 @@ func BackendConfig(ctx context.Context, name string, m configmap.Mapper, ri *Reg
 		// If AutoConfirm is set, choose the default value
 		if ci.AutoConfirm {
 			result := fmt.Sprint(out.Option.Default)
-			Debugf(nil, "Auto confirm is set, choosing default %q for state %q", result, out.State)
+			Debugf(nil, "Auto confirm is set, choosing default %q for state %q, override by setting config parameter %q", result, out.State, out.Option.Name)
 			return ConfigResult(out.State, result)
 		}
 	}

fs/config/config.go

@@ -408,76 +408,117 @@ func getWithDefault(name, key, defaultValue string) string {
 	return value
 }
 
+// UpdateRemoteOpt configures the remote update
+type UpdateRemoteOpt struct {
+	// Treat all passwords as plain that need obscuring
+	Obscure bool `json:"obscure"`
+	// Treat all passwords as obscured
+	NoObscure bool `json:"noObscure"`
+	// Don't interact with the user - return questions
+	NonInteractive bool `json:"nonInteractive"`
+	// If set then supply state and result parameters to continue the process
+	Continue bool `json:"continue"`
+}
+
 // UpdateRemote adds the keyValues passed in to the remote of name.
 // keyValues should be key, value pairs.
-func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscure, noObscure bool) error {
-	if doObscure && noObscure {
-		return errors.New("can't use --obscure and --no-obscure together")
+func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, opt UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
+	if opt.Obscure && opt.NoObscure {
+		return nil, errors.New("can't use --obscure and --no-obscure together")
 	}
-	err := fspath.CheckConfigName(name)
+	err = fspath.CheckConfigName(name)
 	if err != nil {
-		return err
+		return nil, err
+	}
+	interactive := !(opt.NonInteractive || opt.Continue)
+	if interactive {
+		ctx = suppressConfirm(ctx)
 	}
-	ctx = suppressConfirm(ctx)
 
-	// Work out which options need to be obscured
-	needsObscure := map[string]struct{}{}
-	if !noObscure {
-		if fsType := FileGet(name, "type"); fsType != "" {
-			if ri, err := fs.Find(fsType); err != nil {
-				fs.Debugf(nil, "Couldn't find fs for type %q", fsType)
-			} else {
-				for _, opt := range ri.Options {
-					if opt.IsPassword {
-						needsObscure[opt.Name] = struct{}{}
+	fsType := FileGet(name, "type")
+	if fsType == "" {
+		return nil, errors.New("couldn't find type field in config")
+	}
+
+	ri, err := fs.Find(fsType)
+	if err != nil {
+		return nil, errors.Errorf("couldn't find backend for type %q", fsType)
+	}
+
+	if !opt.Continue {
+		// Work out which options need to be obscured
+		needsObscure := map[string]struct{}{}
+		if !opt.NoObscure {
+			for _, option := range ri.Options {
+				if option.IsPassword {
+					needsObscure[option.Name] = struct{}{}
+				}
+			}
+		}
+
+		// Set the config
+		for k, v := range keyValues {
+			vStr := fmt.Sprint(v)
+			// Obscure parameter if necessary
+			if _, ok := needsObscure[k]; ok {
+				_, err := obscure.Reveal(vStr)
+				if err != nil || opt.Obscure {
+					// If error => not already obscured, so obscure it
+					// or we are forced to obscure
+					vStr, err = obscure.Obscure(vStr)
+					if err != nil {
+						return nil, errors.Wrap(err, "UpdateRemote: obscure failed")
 					}
 				}
 			}
-		} else {
-			fs.Debugf(nil, "UpdateRemote: Couldn't find fs type")
+			LoadedData().SetValue(name, k, vStr)
 		}
 	}
 
-	// Set the config
-	for k, v := range keyValues {
-		vStr := fmt.Sprint(v)
-		// Obscure parameter if necessary
-		if _, ok := needsObscure[k]; ok {
-			_, err := obscure.Reveal(vStr)
-			if err != nil || doObscure {
-				// If error => not already obscured, so obscure it
-				// or we are forced to obscure
-				vStr, err = obscure.Obscure(vStr)
-				if err != nil {
-					return errors.Wrap(err, "UpdateRemote: obscure failed")
-				}
+	if interactive {
+		err = RemoteConfig(ctx, name)
+	} else {
+		// Start the config state machine
+		m := fs.ConfigMap(ri, name, nil)
+		in := fs.ConfigIn{}
+		if opt.Continue {
+			if state, ok := keyValues["state"]; ok {
+				in.State = fmt.Sprint(state)
+			} else {
+				return nil, errors.New("UpdateRemote: need state parameter with --continue")
+			}
+			if result, ok := keyValues["result"]; ok {
+				in.Result = fmt.Sprint(result)
+			} else {
+				return nil, errors.New("UpdateRemote: need result parameter with --continue")
 			}
 		}
-		LoadedData().SetValue(name, k, vStr)
+		out, err = fs.BackendConfig(ctx, name, m, ri, in)
 	}
-	err = RemoteConfig(ctx, name)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	SaveConfig()
 	cache.ClearConfig(name) // remove any remotes based on this config from the cache
-	return nil
+	return out, nil
 }
 
 // CreateRemote creates a new remote with name, provider and a list of
 // parameters which are key, value pairs.  If update is set then it
 // adds the new keys rather than replacing all of them.
-func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, doObscure, noObscure bool) error {
-	err := fspath.CheckConfigName(name)
+func CreateRemote(ctx context.Context, name string, provider string, keyValues rc.Params, opts UpdateRemoteOpt) (out *fs.ConfigOut, err error) {
+	err = fspath.CheckConfigName(name)
 	if err != nil {
-		return err
+		return nil, err
+	}
+	if !opts.Continue {
+		// Delete the old config if it exists
+		LoadedData().DeleteSection(name)
+		// Set the type
+		LoadedData().SetValue(name, "type", provider)
 	}
-	// Delete the old config if it exists
-	LoadedData().DeleteSection(name)
-	// Set the type
-	LoadedData().SetValue(name, "type", provider)
 	// Set the remaining values
-	return UpdateRemote(ctx, name, keyValues, doObscure, noObscure)
+	return UpdateRemote(ctx, name, keyValues, opts)
 }
 
 // PasswordRemote adds the keyValues passed in to the remote of name.
@@ -491,7 +532,10 @@ func PasswordRemote(ctx context.Context, name string, keyValues rc.Params) error
 	for k, v := range keyValues {
 		keyValues[k] = obscure.MustObscure(fmt.Sprint(v))
 	}
-	return UpdateRemote(ctx, name, keyValues, false, true)
+	_, err = UpdateRemote(ctx, name, keyValues, UpdateRemoteOpt{
+		NoObscure: true,
+	})
+	return err
 }
 
 // JSONListProviders prints all the providers and options in JSON format

fs/config/rc.go

@@ -3,6 +3,7 @@ package config
 import (
 	"context"
 
+	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/rc"
 )
@@ -112,8 +113,12 @@ func init() {
 			extraHelp = "- type - type of the new remote\n"
 		}
 		if name == "create" || name == "update" {
-			extraHelp += "- obscure - optional bool - forces obscuring of passwords\n"
-			extraHelp += "- noObscure - optional bool - forces passwords not to be obscured\n"
+			extraHelp += `- opt - a dictionary of options to control the configuration
+    - obscure - declare passwords are plain and need obscuring
+    - noObscure - declare passwords are already obscured and don't need obscuring
+    - nonInteractive - don't interact with a user, return questions
+    - continue - continue the config process with an answer
+`
 		}
 		rc.Add(rc.Call{
 			Path:         "config/" + name,
@@ -144,21 +149,47 @@ func rcConfig(ctx context.Context, in rc.Params, what string) (out rc.Params, er
 	if err != nil {
 		return nil, err
 	}
-	doObscure, _ := in.GetBool("obscure")
-	noObscure, _ := in.GetBool("noObscure")
+	var opt UpdateRemoteOpt
+	err = in.GetStruct("opt", &opt)
+	if err != nil && !rc.IsErrParamNotFound(err) {
+		return nil, err
+	}
+	// Backwards compatibility
+	if value, err := in.GetBool("obscure"); err == nil {
+		opt.Obscure = value
+	}
+	if value, err := in.GetBool("noObscure"); err == nil {
+		opt.NoObscure = value
+	}
+	var configOut *fs.ConfigOut
 	switch what {
 	case "create":
-		remoteType, err := in.GetString("type")
-		if err != nil {
-			return nil, err
+		remoteType, typeErr := in.GetString("type")
+		if typeErr != nil {
+			return nil, typeErr
 		}
-		return nil, CreateRemote(ctx, name, remoteType, parameters, doObscure, noObscure)
+		configOut, err = CreateRemote(ctx, name, remoteType, parameters, opt)
 	case "update":
-		return nil, UpdateRemote(ctx, name, parameters, doObscure, noObscure)
+		configOut, err = UpdateRemote(ctx, name, parameters, opt)
 	case "password":
-		return nil, PasswordRemote(ctx, name, parameters)
+		err = PasswordRemote(ctx, name, parameters)
+	default:
+		err = errors.New("unknown rcConfig type")
 	}
-	panic("unknown rcConfig type")
+	if err != nil {
+		return nil, err
+	}
+	if !opt.NonInteractive {
+		return nil, nil
+	}
+	if configOut == nil {
+		configOut = &fs.ConfigOut{}
+	}
+	err = rc.Reshape(&out, configOut)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
 }
 
 func init() {

fs/config/ui.go

@@ -252,7 +252,6 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
 		State: "",
 	}
 	for {
-		fs.Debugf(name, "config: state=%q, result=%q", in.State, in.Result)
 		out, err := fs.BackendConfig(ctx, name, m, ri, in)
 		if err != nil {
 			return err
@@ -266,7 +265,7 @@ func PostConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs.Reg
 		in.State = out.State
 		in.Result = out.Result
 		if out.Option != nil {
-			fs.Debugf(name, "config: reading config item named %q", out.Option.Name)
+			fs.Debugf(name, "config: reading config parameter %q", out.Option.Name)
 			if out.Option.Default == nil {
 				out.Option.Default = ""
 			}

fs/config/ui_test.go

@@ -197,10 +197,15 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
 				break
 			}
 			t.Run(fmt.Sprintf("doObscure=%v,noObscure=%v", doObscure, noObscure), func(t *testing.T) {
-				require.NoError(t, config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
+				opt := config.UpdateRemoteOpt{
+					Obscure:   doObscure,
+					NoObscure: noObscure,
+				}
+				_, err := config.CreateRemote(ctx, "test2", "config_test_remote", rc.Params{
 					"bool": true,
 					"pass": "potato",
-				}, doObscure, noObscure))
+				}, opt)
+				require.NoError(t, err)
 
 				assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
 				assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
@@ -212,11 +217,12 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
 				assert.Equal(t, "potato", gotPw)
 
 				wantPw := obscure.MustObscure("potato2")
-				require.NoError(t, config.UpdateRemote(ctx, "test2", rc.Params{
+				_, err = config.UpdateRemote(ctx, "test2", rc.Params{
 					"bool":  false,
 					"pass":  wantPw,
 					"spare": "spare",
-				}, doObscure, noObscure))
+				}, opt)
+				require.NoError(t, err)
 
 				assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
 				assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))