jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-01-06 02:23:24 +00:00
Code Issues Projects Releases Wiki Activity

Add a download flag to hashsum and related commands to force rclone to download and hash files locally

Browse Source 
This commit modifies the operations.hashSum function by adding an alternate code path. This code path is triggered by passing downloadFlag = True. When activated, rclone will download files from the remote and hash them locally. downloadFlag = False preserves the existing behavior of using the remote to retrieve the hash.

This commit modifies HashLister to support the new hashSum method as well as consolidating the roles of HashLister, HashListerBase64, Md5sum, and Sha1sum.  The printing of hashes from the function defined in HashLister has been revised to work with --progress.  There are light changes to operations.syncFprintf and cmd.startProgress for this.

The unit test operations_test.TestHashSums is modified to support this change and test the download functionality.

The command functions hashsum, md5sum, sha1sum, and dbhashsum are modified to support this change.  A download flag has been added and an output-file flag has been added.  The output-file flag writes hashes to a file instead of stdout to avoid the need to redirect stdout.
This commit is contained in:
lostheli
2020-12-18 07:45:58 -05:00
committed by Nick Craig-Wood
parent ed7af3f370
commit c8cfa43ccc
10 changed files with 319 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/dbhashsum/dbhashsum.go
View File

@@ -2,10 +2,10 @@ package dbhashsum
import (
"context"
"os"
"github.com/rclone/rclone/backend/dropbox"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/hashsum"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
@@ -13,6 +13,8 @@ import (
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
hashsum.AddHashFlags(cmdFlags)
}
var commandDefinition = &cobra.Command{
@@ -23,6 +25,11 @@ Produces a Dropbox hash file for all the objects in the path. The
hashes are calculated according to [Dropbox content hash
rules](https://www.dropbox.com/developers/reference/content-hash).
The output is in the same format as md5sum and sha1sum.
By default, the hash is requested from the remote. If Dropbox hash is
not supported by the remote, no hash will be returned. With the
download flag, the file will be downloaded from the remote and
hashed locally enabling Dropbox hash for any remote.
`,
Hidden: true,
Run: func(command *cobra.Command, args []string) {
@@ -30,7 +37,15 @@ The output is in the same format as md5sum and sha1sum.
fsrc := cmd.NewFsSrc(args)
fs.Logf(nil, `"rclone dbhashsum" is deprecated, use "rclone hashsum %v %s" instead`, dropbox.DbHashType, args[0])
cmd.Run(false, false, command, func() error {
return operations.HashLister(context.Background(), dropbox.DbHashType, fsrc, os.Stdout)
if hashsum.HashsumOutfile == "" {
return operations.HashLister(context.Background(), dropbox.DbHashType, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, nil)
}
output, close, err := hashsum.GetHashsumOutput(hashsum.HashsumOutfile)
if err != nil {
return err
}
defer close()
return operations.HashLister(context.Background(), dropbox.DbHashType, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
})
},
}

55
cmd/hashsum/hashsum.go
View File

@@ -2,25 +2,55 @@ package hashsum
import (
"context"
"errors"
"fmt"
"os"
"github.com/pkg/errors"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Global hashsum flags for reuse in md5sum, sha1sum, and dbhashsum
var (
outputBase64 = false
OutputBase64 = false
DownloadFlag = false
HashsumOutfile = ""
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.BoolVarP(cmdFlags, &outputBase64, "base64", "", outputBase64, "Output base64 encoded hashsum")
AddHashFlags(cmdFlags)
}
// AddHashFlags is a convenience function to add the command flags OutputBase64 and DownloadFlag to hashsum, md5sum, sha1sum, and dbhashsum
func AddHashFlags(cmdFlags *pflag.FlagSet) {
flags.BoolVarP(cmdFlags, &OutputBase64, "base64", "", OutputBase64, "Output base64 encoded hashsum")
flags.StringVarP(cmdFlags, &HashsumOutfile, "output-file", "", HashsumOutfile, "Output hashsums to a file rather than the terminal")
flags.BoolVarP(cmdFlags, &DownloadFlag, "download", "", DownloadFlag, "Download the file and hash it locally; if this flag is not specified, the hash is requested from the remote")
}
// GetHashsumOutput opens and closes the output file when using the output-file flag
func GetHashsumOutput(filename string) (out *os.File, close func(), err error) {
out, err = os.Create(filename)
if err != nil {
err = errors.Wrapf(err, "Failed to open output file %v", filename)
return nil, nil, err
}
close = func() {
err := out.Close()
if err != nil {
fs.Errorf(nil, "Failed to close output file %v: %v", filename, err)
}
}
return out, close, nil
}
var commandDefinition = &cobra.Command{
@@ -31,7 +61,12 @@ Produces a hash file for all the objects in the path using the hash
named. The output is in the same format as the standard
md5sum/sha1sum tool.
Run without a hash to see the list of supported hashes, e.g.
By default, the hash is requested from the remote. If the hash is
not supported by the remote, no hash will be returned. With the
download flag, the file will be downloaded from the remote and
hashed locally enabling any hash for any remote.
Run without a hash to see the list of all supported hashes, e.g.
$ rclone hashsum
Supported hashes are:
@@ -61,11 +96,17 @@ Then
return err
}
fsrc := cmd.NewFsSrc(args[1:])
cmd.Run(false, false, command, func() error {
if outputBase64 {
return operations.HashListerBase64(context.Background(), ht, fsrc, os.Stdout)
if HashsumOutfile == "" {
return operations.HashLister(context.Background(), ht, OutputBase64, DownloadFlag, fsrc, nil)
}
return operations.HashLister(context.Background(), ht, fsrc, os.Stdout)
output, close, err := GetHashsumOutput(HashsumOutfile)
if err != nil {
return err
}
defer close()
return operations.HashLister(context.Background(), ht, OutputBase64, DownloadFlag, fsrc, output)
})
return nil
},

20
cmd/md5sum/md5sum.go
View File

@@ -2,15 +2,18 @@ package md5sum
import (
"context"
"os"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/hashsum"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
hashsum.AddHashFlags(cmdFlags)
}
var commandDefinition = &cobra.Command{
@@ -19,12 +22,25 @@ var commandDefinition = &cobra.Command{
Long: `
Produces an md5sum file for all the objects in the path. This
is in the same format as the standard md5sum tool produces.
By default, the hash is requested from the remote. If MD5 is
not supported by the remote, no hash will be returned. With the
download flag, the file will be downloaded from the remote and
hashed locally enabling MD5 for any remote.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
return operations.Md5sum(context.Background(), fsrc, os.Stdout)
if hashsum.HashsumOutfile == "" {
return operations.HashLister(context.Background(), hash.MD5, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, nil)
}
output, close, err := hashsum.GetHashsumOutput(hashsum.HashsumOutfile)
if err != nil {
return err
}
defer close()
return operations.HashLister(context.Background(), hash.MD5, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
})
},
}

10
cmd/progress.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/terminal"
)
@@ -28,6 +29,8 @@ const (
func startProgress() func() {
stopStats := make(chan struct{})
oldLogPrint := fs.LogPrint
oldSyncPrint := operations.SyncPrintf
if !log.Redirected() {
// Intercept the log calls if not logging to file or syslog
fs.LogPrint = func(level fs.LogLevel, text string) {
@@ -35,6 +38,12 @@ func startProgress() func() {
}
}
// Intercept output from functions such as HashLister to stdout
operations.SyncPrintf = func(format string, a ...interface{}) {
printProgress(fmt.Sprintf(format, a...))
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
@@ -52,6 +61,7 @@ func startProgress() func() {
ticker.Stop()
printProgress("")
fs.LogPrint = oldLogPrint
operations.SyncPrintf = oldSyncPrint
fmt.Println("")
return
}

20
cmd/sha1sum/sha1sum.go
View File

@@ -2,15 +2,18 @@ package sha1sum
import (
"context"
"os"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/hashsum"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
hashsum.AddHashFlags(cmdFlags)
}
var commandDefinition = &cobra.Command{
@@ -19,12 +22,25 @@ var commandDefinition = &cobra.Command{
Long: `
Produces an sha1sum file for all the objects in the path. This
is in the same format as the standard sha1sum tool produces.
By default, the hash is requested from the remote. If SHA-1 is
not supported by the remote, no hash will be returned. With the
download flag, the file will be downloaded from the remote and
hashed locally enabling SHA-1 for any remote.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
return operations.Sha1sum(context.Background(), fsrc, os.Stdout)
if hashsum.HashsumOutfile == "" {
return operations.HashLister(context.Background(), hash.SHA1, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, nil)
}
output, close, err := hashsum.GetHashsumOutput(hashsum.HashsumOutfile)
if err != nil {
return err
}
defer close()
return operations.HashLister(context.Background(), hash.SHA1, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
})
},
}