test speed: add command to test a specified remotes speed

Run speed test to try and work in a given time budget, uploading
randomly created files to the remote then downloading them again.

Fixes #3198

cmd/test/makefiles/makefiles.go

@@ -61,8 +61,12 @@ func init() {
 	test.Command.AddCommand(makefileCmd)
 	makefileFlags := makefileCmd.Flags()
 
-	// Common flags to makefiles and makefile
+	addCommonFlags(makefilesFlags)
-	for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {
+	addCommonFlags(makefileFlags)
+}
+
+// Common flags for makefiles and makefile
+func addCommonFlags(f *pflag.FlagSet) {
 	flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")
 	flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")
 	flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")
@@ -70,7 +74,6 @@ func init() {
 	flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")
 	flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")
 }
-}
 
 var makefilesCmd = &cobra.Command{
 	Use:   "makefiles <dir>",
@@ -123,10 +126,15 @@ var makefileCmd = &cobra.Command{
 		if err != nil {
 			fs.Fatalf(nil, "Failed to parse size %q: %v", args[0], err)
 		}
+		makefiles(size, args[1:])
+	},
+}
+
+func makefiles(size fs.SizeSuffix, files []string) {
 	start := time.Now()
-		fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)
+	fs.Logf(nil, "Creating %d files of size %v.", len(files), size)
 	totalBytes := int64(0)
-		for _, filePath := range args[1:] {
+	for _, filePath := range files {
 		dir := filepath.Dir(filePath)
 		name := filepath.Base(filePath)
 		writeFile(dir, name, int64(size))
@@ -134,7 +142,6 @@ var makefileCmd = &cobra.Command{
 	}
 	dt := time.Since(start)
 	fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
-	},
 }
 
 func bool2int(b bool) int {

cmd/test/makefiles/speed.go

@@ -0,0 +1,235 @@
+package makefiles
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"time"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/test"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
+	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/spf13/cobra"
+)
+
+var (
+	// Flags
+	testTime = fs.Duration(15 * time.Second)
+	fcap     = 100
+	small    = fs.SizeSuffix(1024)
+	medium   = fs.SizeSuffix(10 * 1024 * 1024)
+	large    = fs.SizeSuffix(1024 * 1024 * 1024)
+	useJSON  = false
+)
+
+func init() {
+	test.Command.AddCommand(speedCmd)
+
+	speedFlags := speedCmd.Flags()
+	flags.FVarP(speedFlags, &testTime, "test-time", "", "Length for each test to run", "")
+	flags.IntVarP(speedFlags, &fcap, "file-cap", "", fcap, "Maximum number of files to use in each test", "")
+	flags.FVarP(speedFlags, &small, "small", "", "Size of small files", "")
+	flags.FVarP(speedFlags, &medium, "medium", "", "Size of medium files", "")
+	flags.FVarP(speedFlags, &large, "large", "", "Size of large files", "")
+	flags.BoolVarP(speedFlags, &useJSON, "json", "", useJSON, "Output only results in JSON format", "")
+
+	addCommonFlags(speedFlags)
+}
+
+func logf(text string, args ...any) {
+	if !useJSON {
+		fmt.Printf(text, args...)
+	}
+}
+
+var speedCmd = &cobra.Command{
+	Use:   "speed <remote> [flags]",
+	Short: `Run a speed test to the remote`,
+	Long: `Run a speed test to the remote.
+
+	This command runs a series of uploads and downloads to the remote, measuring
+	and printing the speed of each test using varying file sizes and numbers of
+	files.
+
+	Test time can be innaccurate with small file caps and large files. As it
+	uses the results of an initial test to determine how many files to use in
+	each subsequent test.
+
+	It is recommended to use -q flag for a simpler output. e.g.:
+	
+	    rlone test speed remote: -q
+
+	**NB** This command will create and delete files on the remote in a randomly
+	named directory which should be tidied up after.
+
+	You can use the --json flag to only print the results in JSON format.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.72",
+	},
+	RunE: func(command *cobra.Command, args []string) error {
+		ctx := command.Context()
+		cmd.CheckArgs(1, 1, command, args)
+		commonInit()
+
+		// initial test
+		size := fs.SizeSuffix(1024 * 1024)
+		logf("Running initial test for 4 files of size %v\n", size)
+		stats, err := speedTest(ctx, 4, size, args[0])
+		if err != nil {
+			return fmt.Errorf("speed test failed: %w", err)
+		}
+
+		var results []*Stats
+
+		// main tests
+		logf("\nTest Time: %v, File cap: %d\n", testTime, fcap)
+		for _, size := range []fs.SizeSuffix{small, medium, large} {
+			numberOfFilesUpload := int((float64(stats.Upload.Speed) * time.Duration(testTime).Seconds()) / float64(size))
+			numberOfFilesDownload := int((float64(stats.Download.Speed) * time.Duration(testTime).Seconds()) / float64(size))
+			numberOfFiles := min(numberOfFilesUpload, numberOfFilesDownload)
+
+			logf("\nNumber of files for upload and download: %v\n", numberOfFiles)
+			if numberOfFiles < 1 {
+				logf("Skipping test for file size %v as calculated number of files is 0\n", size)
+				continue
+			} else if numberOfFiles > fcap {
+				numberOfFiles = fcap
+				logf("Capping test for file size %v to %v files\n", size, fcap)
+			}
+
+			logf("Running test for %d files of size %v\n", numberOfFiles, size)
+			s, err := speedTest(ctx, numberOfFiles, size, args[0])
+			if err != nil {
+				return fmt.Errorf("speed test failed: %w", err)
+			}
+			results = append(results, s)
+
+		}
+
+		if useJSON {
+			b, err := json.MarshalIndent(results, "", "  ")
+			if err != nil {
+				return fmt.Errorf("failed to marshal results to JSON: %w", err)
+			}
+			fmt.Println(string(b))
+		}
+
+		return nil
+	},
+}
+
+// Stats of a speed test
+type Stats struct {
+	Size          fs.SizeSuffix
+	NumberOfFiles int
+	Upload        TestResult
+	Download      TestResult
+}
+
+// TestResult of a speed test operation
+type TestResult struct {
+	Bytes    int64
+	Duration time.Duration
+	Speed    fs.SizeSuffix
+}
+
+// measures stats for speedTest operations
+func measure(desc string, f func() error, size fs.SizeSuffix, numberOfFiles int, tr *TestResult) error {
+	start := time.Now()
+	err := f()
+	dt := time.Since(start)
+	if err != nil {
+		return err
+	}
+	tr.Duration = dt
+	tr.Bytes = int64(size) * int64(numberOfFiles)
+	tr.Speed = fs.SizeSuffix(float64(tr.Bytes) / dt.Seconds())
+	logf("%-20s: %vB in %v at %vB/s\n", desc, tr.Bytes, dt.Round(time.Millisecond), tr.Speed)
+	return err
+}
+
+func speedTest(ctx context.Context, numberOfFiles int, size fs.SizeSuffix, remote string) (*Stats, error) {
+	stats := Stats{
+		Size:          size,
+		NumberOfFiles: numberOfFiles,
+	}
+
+	tempDirName := "rclone-speed-test-" + random.String(8)
+	tempDirPath := path.Join(remote, tempDirName)
+	fremote := cmd.NewFsDir([]string{tempDirPath})
+	aErr := io.EOF
+	defer atexit.OnError(&aErr, func() {
+		err := operations.Purge(ctx, fremote, "")
+		if err != nil {
+			fs.Debugf(fremote, "Failed to remove temp dir %q: %v", tempDirPath, err)
+		}
+	})()
+
+	flocalDir, err := os.MkdirTemp("", "rclone-speedtest-local-")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create local temp dir: %w", err)
+	}
+	defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(flocalDir) })()
+
+	flocal, err := cache.Get(ctx, flocalDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create local fs: %w", err)
+	}
+
+	fdownloadDir, err := os.MkdirTemp("", "rclone-speedtest-download-")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create download temp dir: %w", err)
+	}
+	defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(fdownloadDir) })()
+
+	fdownload, err := cache.Get(ctx, fdownloadDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create download fs: %w", err)
+	}
+
+	// make the largest amount of files we will need
+	files := make([]string, numberOfFiles)
+	for i := range files {
+		files[i] = path.Join(flocalDir, fmt.Sprintf("file%03d-%v.bin", i, size))
+	}
+	makefiles(size, files)
+
+	// upload files
+	err = measure("Upload", func() error {
+		return sync.CopyDir(ctx, fremote, flocal, false)
+	}, size, numberOfFiles, &stats.Upload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to Copy to remote: %w", err)
+	}
+
+	// download files
+	err = measure("Download", func() error {
+		return sync.CopyDir(ctx, fdownload, fremote, false)
+	}, size, numberOfFiles, &stats.Download)
+	if err != nil {
+		return nil, fmt.Errorf("failed to Copy from remote: %w", err)
+	}
+
+	// check files
+	opt := operations.CheckOpt{
+		Fsrc:   flocal,
+		Fdst:   fdownload,
+		OneWay: false,
+	}
+	logf("Checking file integrity\n")
+	err = operations.CheckDownload(ctx, &opt)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check redownloaded files were identical: %w", err)
+	}
+
+	return &stats, nil
+}