mirror of
https://github.com/rclone/rclone.git
synced 2025-12-06 00:03:32 +00:00
test speed: add command to test a specified remotes speed
Run speed test to try and work in a given time budget, uploading randomly created files to the remote then downloading them again. Fixes #3198
This commit is contained in:
dougal
GitHub
@@ -61,15 +61,18 @@ func init() {
|
||||
test.Command.AddCommand(makefileCmd)
|
||||
makefileFlags := makefileCmd.Flags()
|
||||
|
||||
// Common flags to makefiles and makefile
|
||||
for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {
|
||||
flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")
|
||||
flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")
|
||||
flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")
|
||||
flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")
|
||||
flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")
|
||||
flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")
|
||||
}
|
||||
addCommonFlags(makefilesFlags)
|
||||
addCommonFlags(makefileFlags)
|
||||
}
|
||||
|
||||
// Common flags for makefiles and makefile
|
||||
func addCommonFlags(f *pflag.FlagSet) {
|
||||
flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)", "")
|
||||
flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00", "")
|
||||
flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)", "")
|
||||
flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only", "")
|
||||
flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern", "")
|
||||
flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern", "")
|
||||
}
|
||||
|
||||
var makefilesCmd = &cobra.Command{
|
||||
@@ -123,20 +126,24 @@ var makefileCmd = &cobra.Command{
|
||||
if err != nil {
|
||||
fs.Fatalf(nil, "Failed to parse size %q: %v", args[0], err)
|
||||
}
|
||||
start := time.Now()
|
||||
fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)
|
||||
totalBytes := int64(0)
|
||||
for _, filePath := range args[1:] {
|
||||
dir := filepath.Dir(filePath)
|
||||
name := filepath.Base(filePath)
|
||||
writeFile(dir, name, int64(size))
|
||||
totalBytes += int64(size)
|
||||
}
|
||||
dt := time.Since(start)
|
||||
fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
makefiles(size, args[1:])
|
||||
},
|
||||
}
|
||||
|
||||
func makefiles(size fs.SizeSuffix, files []string) {
|
||||
start := time.Now()
|
||||
fs.Logf(nil, "Creating %d files of size %v.", len(files), size)
|
||||
totalBytes := int64(0)
|
||||
for _, filePath := range files {
|
||||
dir := filepath.Dir(filePath)
|
||||
name := filepath.Base(filePath)
|
||||
writeFile(dir, name, int64(size))
|
||||
totalBytes += int64(size)
|
||||
}
|
||||
dt := time.Since(start)
|
||||
fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
}
|
||||
|
||||
func bool2int(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
|
||||
235
cmd/test/makefiles/speed.go
Normal file
235
cmd/test/makefiles/speed.go
Normal file
@@ -0,0 +1,235 @@
|
||||
package makefiles
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/test"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
// Flags
|
||||
testTime = fs.Duration(15 * time.Second)
|
||||
fcap = 100
|
||||
small = fs.SizeSuffix(1024)
|
||||
medium = fs.SizeSuffix(10 * 1024 * 1024)
|
||||
large = fs.SizeSuffix(1024 * 1024 * 1024)
|
||||
useJSON = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(speedCmd)
|
||||
|
||||
speedFlags := speedCmd.Flags()
|
||||
flags.FVarP(speedFlags, &testTime, "test-time", "", "Length for each test to run", "")
|
||||
flags.IntVarP(speedFlags, &fcap, "file-cap", "", fcap, "Maximum number of files to use in each test", "")
|
||||
flags.FVarP(speedFlags, &small, "small", "", "Size of small files", "")
|
||||
flags.FVarP(speedFlags, &medium, "medium", "", "Size of medium files", "")
|
||||
flags.FVarP(speedFlags, &large, "large", "", "Size of large files", "")
|
||||
flags.BoolVarP(speedFlags, &useJSON, "json", "", useJSON, "Output only results in JSON format", "")
|
||||
|
||||
addCommonFlags(speedFlags)
|
||||
}
|
||||
|
||||
func logf(text string, args ...any) {
|
||||
if !useJSON {
|
||||
fmt.Printf(text, args...)
|
||||
}
|
||||
}
|
||||
|
||||
var speedCmd = &cobra.Command{
|
||||
Use: "speed <remote> [flags]",
|
||||
Short: `Run a speed test to the remote`,
|
||||
Long: `Run a speed test to the remote.
|
||||
|
||||
This command runs a series of uploads and downloads to the remote, measuring
|
||||
and printing the speed of each test using varying file sizes and numbers of
|
||||
files.
|
||||
|
||||
Test time can be innaccurate with small file caps and large files. As it
|
||||
uses the results of an initial test to determine how many files to use in
|
||||
each subsequent test.
|
||||
|
||||
It is recommended to use -q flag for a simpler output. e.g.:
|
||||
|
||||
rlone test speed remote: -q
|
||||
|
||||
**NB** This command will create and delete files on the remote in a randomly
|
||||
named directory which should be tidied up after.
|
||||
|
||||
You can use the --json flag to only print the results in JSON format.`,
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.72",
|
||||
},
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
ctx := command.Context()
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
commonInit()
|
||||
|
||||
// initial test
|
||||
size := fs.SizeSuffix(1024 * 1024)
|
||||
logf("Running initial test for 4 files of size %v\n", size)
|
||||
stats, err := speedTest(ctx, 4, size, args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("speed test failed: %w", err)
|
||||
}
|
||||
|
||||
var results []*Stats
|
||||
|
||||
// main tests
|
||||
logf("\nTest Time: %v, File cap: %d\n", testTime, fcap)
|
||||
for _, size := range []fs.SizeSuffix{small, medium, large} {
|
||||
numberOfFilesUpload := int((float64(stats.Upload.Speed) * time.Duration(testTime).Seconds()) / float64(size))
|
||||
numberOfFilesDownload := int((float64(stats.Download.Speed) * time.Duration(testTime).Seconds()) / float64(size))
|
||||
numberOfFiles := min(numberOfFilesUpload, numberOfFilesDownload)
|
||||
|
||||
logf("\nNumber of files for upload and download: %v\n", numberOfFiles)
|
||||
if numberOfFiles < 1 {
|
||||
logf("Skipping test for file size %v as calculated number of files is 0\n", size)
|
||||
continue
|
||||
} else if numberOfFiles > fcap {
|
||||
numberOfFiles = fcap
|
||||
logf("Capping test for file size %v to %v files\n", size, fcap)
|
||||
}
|
||||
|
||||
logf("Running test for %d files of size %v\n", numberOfFiles, size)
|
||||
s, err := speedTest(ctx, numberOfFiles, size, args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("speed test failed: %w", err)
|
||||
}
|
||||
results = append(results, s)
|
||||
|
||||
}
|
||||
|
||||
if useJSON {
|
||||
b, err := json.MarshalIndent(results, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal results to JSON: %w", err)
|
||||
}
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// Stats of a speed test
|
||||
type Stats struct {
|
||||
Size fs.SizeSuffix
|
||||
NumberOfFiles int
|
||||
Upload TestResult
|
||||
Download TestResult
|
||||
}
|
||||
|
||||
// TestResult of a speed test operation
|
||||
type TestResult struct {
|
||||
Bytes int64
|
||||
Duration time.Duration
|
||||
Speed fs.SizeSuffix
|
||||
}
|
||||
|
||||
// measures stats for speedTest operations
|
||||
func measure(desc string, f func() error, size fs.SizeSuffix, numberOfFiles int, tr *TestResult) error {
|
||||
start := time.Now()
|
||||
err := f()
|
||||
dt := time.Since(start)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tr.Duration = dt
|
||||
tr.Bytes = int64(size) * int64(numberOfFiles)
|
||||
tr.Speed = fs.SizeSuffix(float64(tr.Bytes) / dt.Seconds())
|
||||
logf("%-20s: %vB in %v at %vB/s\n", desc, tr.Bytes, dt.Round(time.Millisecond), tr.Speed)
|
||||
return err
|
||||
}
|
||||
|
||||
func speedTest(ctx context.Context, numberOfFiles int, size fs.SizeSuffix, remote string) (*Stats, error) {
|
||||
stats := Stats{
|
||||
Size: size,
|
||||
NumberOfFiles: numberOfFiles,
|
||||
}
|
||||
|
||||
tempDirName := "rclone-speed-test-" + random.String(8)
|
||||
tempDirPath := path.Join(remote, tempDirName)
|
||||
fremote := cmd.NewFsDir([]string{tempDirPath})
|
||||
aErr := io.EOF
|
||||
defer atexit.OnError(&aErr, func() {
|
||||
err := operations.Purge(ctx, fremote, "")
|
||||
if err != nil {
|
||||
fs.Debugf(fremote, "Failed to remove temp dir %q: %v", tempDirPath, err)
|
||||
}
|
||||
})()
|
||||
|
||||
flocalDir, err := os.MkdirTemp("", "rclone-speedtest-local-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create local temp dir: %w", err)
|
||||
}
|
||||
defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(flocalDir) })()
|
||||
|
||||
flocal, err := cache.Get(ctx, flocalDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create local fs: %w", err)
|
||||
}
|
||||
|
||||
fdownloadDir, err := os.MkdirTemp("", "rclone-speedtest-download-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create download temp dir: %w", err)
|
||||
}
|
||||
defer atexit.OnError(&aErr, func() { _ = os.RemoveAll(fdownloadDir) })()
|
||||
|
||||
fdownload, err := cache.Get(ctx, fdownloadDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create download fs: %w", err)
|
||||
}
|
||||
|
||||
// make the largest amount of files we will need
|
||||
files := make([]string, numberOfFiles)
|
||||
for i := range files {
|
||||
files[i] = path.Join(flocalDir, fmt.Sprintf("file%03d-%v.bin", i, size))
|
||||
}
|
||||
makefiles(size, files)
|
||||
|
||||
// upload files
|
||||
err = measure("Upload", func() error {
|
||||
return sync.CopyDir(ctx, fremote, flocal, false)
|
||||
}, size, numberOfFiles, &stats.Upload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to Copy to remote: %w", err)
|
||||
}
|
||||
|
||||
// download files
|
||||
err = measure("Download", func() error {
|
||||
return sync.CopyDir(ctx, fdownload, fremote, false)
|
||||
}, size, numberOfFiles, &stats.Download)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to Copy from remote: %w", err)
|
||||
}
|
||||
|
||||
// check files
|
||||
opt := operations.CheckOpt{
|
||||
Fsrc: flocal,
|
||||
Fdst: fdownload,
|
||||
OneWay: false,
|
||||
}
|
||||
logf("Checking file integrity\n")
|
||||
err = operations.CheckDownload(ctx, &opt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check redownloaded files were identical: %w", err)
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
Reference in New Issue
Block a user