diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go index 40390a663..921d708cf 100644 --- a/cmd/test/makefiles/makefiles.go +++ b/cmd/test/makefiles/makefiles.go @@ -61,15 +61,18 @@ func init() { test.Command.AddCommand(makefileCmd) makefileFlags := makefileCmd.Flags() - // Common flags to makefiles and makefile - for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} { - flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "") - flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "") - flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "") - flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "") - flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "") - flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "") - } + addCommonFlags(makefilesFlags) + addCommonFlags(makefileFlags) +} + +// Common flags for makefiles and makefile +func addCommonFlags(f *pflag.FlagSet) { + flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "") + flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "") + flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "") + flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "") + flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "") + flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "") } var makefilesCmd = &cobra.Command{ @@ -123,20 +126,24 @@ var makefileCmd = &cobra.Command{ if err != nil { fs.Fatalf(nil, "Failed to parse size %q: %v", args[0], err) } - start := time.Now() - fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size) - totalBytes := int64(0) - for _, filePath := range args[1:] { - dir := filepath.Dir(filePath) - name := filepath.Base(filePath) - writeFile(dir, name, int64(size)) - totalBytes += int64(size) - } - dt := time.Since(start) - fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt))) + makefiles(size, args[1:]) }, } +func makefiles(size fs.SizeSuffix, files []string) { + start := time.Now() + fs.Logf(nil, "Creating %d files of size %v.", len(files), size) + totalBytes := int64(0) + for _, filePath := range files { + dir := filepath.Dir(filePath) + name := filepath.Base(filePath) + writeFile(dir, name, int64(size)) + totalBytes += int64(size) + } + dt := time.Since(start) + fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt))) +} + func bool2int(b bool) int { if b { return 1 diff --git a/cmd/test/makefiles/speed.go b/cmd/test/makefiles/speed.go new file mode 100644 index 000000000..fc7edeb6d --- /dev/null +++ b/cmd/test/makefiles/speed.go @@ -0,0 +1,235 @@ +package makefiles + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path" + "time" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/cmd/test" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/random" + "github.com/spf13/cobra" +) + +var ( + // Flags + testTime = fs.Duration(15 * time.Second) + fcap = 100 + small = fs.SizeSuffix(1024) + medium = fs.SizeSuffix(10 * 1024 * 1024) + large = fs.SizeSuffix(1024 * 1024 * 1024) + useJSON = false +) + +func init() { + test.Command.AddCommand(speedCmd) + + speedFlags := speedCmd.Flags() + flags.FVarP(speedFlags, &testTime, "test-time", "", "Length for each test to run", "") + flags.IntVarP(speedFlags, &fcap, "file-cap", "", fcap, "Maximum number of files to use in each test", "") + flags.FVarP(speedFlags, &small, "small", "", "Size of small files", "") + flags.FVarP(speedFlags, &medium, "medium", "", "Size of medium files", "") + flags.FVarP(speedFlags, &large, "large", "", "Size of large files", "") + flags.BoolVarP(speedFlags, &useJSON, "json", "", useJSON, "Output only results in JSON format", "") + + addCommonFlags(speedFlags) +} + +func logf(text string, args ...any) { + if !useJSON { + fmt.Printf(text, args...) + } +} + +var speedCmd = &cobra.Command{ + Use: "speed [flags]", + Short: `Run a speed test to the remote`, + Long: `Run a speed test to the remote. + + This command runs a series of uploads and downloads to the remote, measuring + and printing the speed of each test using varying file sizes and numbers of + files. + + Test time can be innaccurate with small file caps and large files. As it + uses the results of an initial test to determine how many files to use in + each subsequent test. + + It is recommended to use -q flag for a simpler output. e.g.: + + rlone test speed remote: -q + + **NB** This command will create and delete files on the remote in a randomly + named directory which should be tidied up after. + + You can use the --json flag to only print the results in JSON format.`, + Annotations: map[string]string{ + "versionIntroduced": "v1.72", + }, + RunE: func(command *cobra.Command, args []string) error { + ctx := command.Context() + cmd.CheckArgs(1, 1, command, args) + commonInit() + + // initial test + size := fs.SizeSuffix(1024 * 1024) + logf("Running initial test for 4 files of size %v\n", size) + stats, err := speedTest(ctx, 4, size, args[0]) + if err != nil { + return fmt.Errorf("speed test failed: %w", err) + } + + var results []*Stats + + // main tests + logf("\nTest Time: %v, File cap: %d\n", testTime, fcap) + for _, size := range []fs.SizeSuffix{small, medium, large} { + numberOfFilesUpload := int((float64(stats.Upload.Speed) * time.Duration(testTime).Seconds()) / float64(size)) + numberOfFilesDownload := int((float64(stats.Download.Speed) * time.Duration(testTime).Seconds()) / float64(size)) + numberOfFiles := min(numberOfFilesUpload, numberOfFilesDownload) + + logf("\nNumber of files for upload and download: %v\n", numberOfFiles) + if numberOfFiles < 1 { + logf("Skipping test for file size %v as calculated number of files is 0\n", size) + continue + } else if numberOfFiles > fcap { + numberOfFiles = fcap + logf("Capping test for file size %v to %v files\n", size, fcap) + } + + logf("Running test for %d files of size %v\n", numberOfFiles, size) + s, err := speedTest(ctx, numberOfFiles, size, args[0]) + if err != nil { + return fmt.Errorf("speed test failed: %w", err) + } + results = append(results, s) + + } + + if useJSON { + b, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal results to JSON: %w", err) + } + fmt.Println(string(b)) + } + + return nil + }, +} + +// Stats of a speed test +type Stats struct { + Size fs.SizeSuffix + NumberOfFiles int + Upload TestResult + Download TestResult +} + +// TestResult of a speed test operation +type TestResult struct { + Bytes int64 + Duration time.Duration + Speed fs.SizeSuffix +} + +// measures stats for speedTest operations +func measure(desc string, f func() error, size fs.SizeSuffix, numberOfFiles int, tr *TestResult) error { + start := time.Now() + err := f() + dt := time.Since(start) + if err != nil { + return err + } + tr.Duration = dt + tr.Bytes = int64(size) * int64(numberOfFiles) + tr.Speed = fs.SizeSuffix(float64(tr.Bytes) / dt.Seconds()) + logf("%-20s: %vB in %v at %vB/s\n", desc, tr.Bytes, dt.Round(time.Millisecond), tr.Speed) + return err +} + +func speedTest(ctx context.Context, numberOfFiles int, size fs.SizeSuffix, remote string) (*Stats, error) { + stats := Stats{ + Size: size, + NumberOfFiles: numberOfFiles, + } + + tempDirName := "rclone-speed-test-" + random.String(8) + tempDirPath := path.Join(remote, tempDirName) + fremote := cmd.NewFsDir([]string{tempDirPath}) + aErr := io.EOF + defer atexit.OnError(&aErr, func() { + err := operations.Purge(ctx, fremote, "") + if err != nil { + fs.Debugf(fremote, "Failed to remove temp dir %q: %v", tempDirPath, err) + } + })() + + flocalDir, err := os.MkdirTemp("", "rclone-speedtest-local-") + if err != nil { + return nil, fmt.Errorf("failed to create local temp dir: %w", err) + } + defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(flocalDir) })() + + flocal, err := cache.Get(ctx, flocalDir) + if err != nil { + return nil, fmt.Errorf("failed to create local fs: %w", err) + } + + fdownloadDir, err := os.MkdirTemp("", "rclone-speedtest-download-") + if err != nil { + return nil, fmt.Errorf("failed to create download temp dir: %w", err) + } + defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(fdownloadDir) })() + + fdownload, err := cache.Get(ctx, fdownloadDir) + if err != nil { + return nil, fmt.Errorf("failed to create download fs: %w", err) + } + + // make the largest amount of files we will need + files := make([]string, numberOfFiles) + for i := range files { + files[i] = path.Join(flocalDir, fmt.Sprintf("file%03d-%v.bin", i, size)) + } + makefiles(size, files) + + // upload files + err = measure("Upload", func() error { + return sync.CopyDir(ctx, fremote, flocal, false) + }, size, numberOfFiles, &stats.Upload) + if err != nil { + return nil, fmt.Errorf("failed to Copy to remote: %w", err) + } + + // download files + err = measure("Download", func() error { + return sync.CopyDir(ctx, fdownload, fremote, false) + }, size, numberOfFiles, &stats.Download) + if err != nil { + return nil, fmt.Errorf("failed to Copy from remote: %w", err) + } + + // check files + opt := operations.CheckOpt{ + Fsrc: flocal, + Fdst: fdownload, + OneWay: false, + } + logf("Checking file integrity\n") + err = operations.CheckDownload(ctx, &opt) + if err != nil { + return nil, fmt.Errorf("failed to check redownloaded files were identical: %w", err) + } + + return &stats, nil +}