press: rebase and fix the broken tests

... add additional error handling for metadata upload
... fix metadata wrapping
... make sure not to leave any orphaned data when overwriting existing files with Put, Copy or Move
... restructure some minor things to make them clearer
... remove xz compression

backend/all/all.go

@@ -28,6 +28,7 @@ import (
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
+	_ "github.com/rclone/rclone/backend/press"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"

backend/press/alg_exec.go

@@ -1,98 +0,0 @@
-package press
-
-// This file implements shell exec algorithms that require binaries.
-import (
-	"bytes"
-	"io"
-	"os/exec"
-)
-
-// XZ command
-const xzcommand = "xz" // Name of xz binary (if available)
-
-// ExecHeader - Header we add to an exec file. We don't need this.
-var ExecHeader = []byte{}
-
-// Function that checks whether XZ is present in the system
-func checkXZ() bool {
-	_, err := exec.LookPath("xz")
-	if err != nil {
-		return false
-	}
-	return true
-}
-
-// Function that gets binary paths if needed
-func getBinPaths(c *Compression, mode int) (err error) {
-	err = nil
-	if mode == XZMin || mode == XZDefault {
-		c.BinPath, err = exec.LookPath(xzcommand)
-	}
-	return err
-}
-
-// Function that compresses a block using a shell command without wrapping in gzip. Requires an binary corresponding with the command.
-func (c *Compression) compressBlockExec(in []byte, out io.Writer, binaryPath string, args []string) (compressedSize uint32, uncompressedSize int64, err error) {
-	// Initialize compression subprocess
-	subprocess := exec.Command(binaryPath, args...)
-	stdin, err := subprocess.StdinPipe()
-	if err != nil {
-		return 0, 0, err
-	}
-
-	// Run subprocess that creates compressed file
-	stdinError := make(chan error)
-	go func() {
-		_, err := stdin.Write(in)
-		_ = stdin.Close()
-		stdinError <- err
-	}()
-
-	// Get output
-	output, err := subprocess.Output()
-	if err != nil {
-		return 0, 0, err
-	}
-
-	// Copy over
-	n, err := io.Copy(out, bytes.NewReader(output))
-	if err != nil {
-		return uint32(n), int64(len(in)), err
-	}
-
-	// Check if there was an error and return
-	err = <-stdinError
-
-	return uint32(n), int64(len(in)), err
-}
-
-// Utility function to decompress a block range using a shell command which wasn't wrapped in gzip
-func decompressBlockRangeExec(in io.Reader, out io.Writer, binaryPath string, args []string) (n int, err error) {
-	// Decompress actual compression
-	// Initialize decompression subprocess
-	subprocess := exec.Command(binaryPath, args...)
-	stdin, err := subprocess.StdinPipe()
-	if err != nil {
-		return 0, err
-	}
-
-	// Run subprocess that copies over compressed block
-	stdinError := make(chan error)
-	go func() {
-		_, err := io.Copy(stdin, in)
-		_ = stdin.Close()
-		stdinError <- err
-	}()
-
-	// Get output, copy, and return
-	output, err := subprocess.Output()
-	if err != nil {
-		return 0, err
-	}
-	n64, err := io.Copy(out, bytes.NewReader(output))
-	if err != nil {
-		return int(n64), err
-	}
-	err = <-stdinError
-	return int(n64), err
-}

backend/press/compression.go

@@ -36,8 +36,6 @@ const (
 	GzipMax      = 3
 	LZ4          = 4
 	Snappy       = 5
-	XZMin        = 6
-	XZDefault    = 7
 )
 
 // Errors
@@ -72,10 +70,6 @@ func NewCompressionPreset(preset string) (*Compression, error) {
 		return NewCompression(GzipMin, 131070) // GZIP-min compression (fast)
 	case "gzip-default":
 		return NewCompression(GzipDefault, 131070) // GZIP-default compression (medium)
-	case "xz-min":
-		return NewCompression(XZMin, 524288) // XZ-min compression (slow)
-	case "xz-default":
-		return NewCompression(XZDefault, 1048576) // XZ-default compression (very slow)
 	}
 	return nil, errors.New("Compression mode doesn't exist")
 }
@@ -93,10 +87,6 @@ func NewCompressionPresetNumber(preset int) (*Compression, error) {
 		return NewCompression(GzipMin, 131070) // GZIP-min compression (fast)
 	case GzipDefault:
 		return NewCompression(GzipDefault, 131070) // GZIP-default compression (medium)
-	case XZMin:
-		return NewCompression(XZMin, 524288) // XZ-min compression (slow)
-	case XZDefault:
-		return NewCompression(XZDefault, 1048576) // XZ-default compression (very slow)
 	}
 	return nil, errors.New("Compression mode doesn't exist")
 }
@@ -115,8 +105,6 @@ func NewCompressionAdvanced(mode int, bs uint32, hb int64, threads int, mcr floa
 	c.HeuristicBytes = hb
 	c.NumThreads = threads
 	c.MaxCompressionRatio = mcr
-	// Get binary path if needed
-	err = getBinPaths(c, mode)
 	return c, err
 }
 
@@ -131,8 +119,6 @@ func (c *Compression) GetFileExtension() string {
 	switch c.CompressionMode {
 	case GzipStore, GzipMin, GzipDefault, GzipMax:
 		return ".gz"
-	case XZMin, XZDefault:
-		return ".xzgz"
 	case LZ4:
 		return ".lz4"
 	case Snappy:
@@ -142,7 +128,6 @@ func (c *Compression) GetFileExtension() string {
 }
 
 // GetFileCompressionInfo gets a file extension along with compressibility of file
-// It is currently not being used but may be usable in the future.
 func (c *Compression) GetFileCompressionInfo(reader io.Reader) (compressable bool, extension string, err error) {
 	// Use our compression algorithm to do a heuristic on the first few bytes
 	var emulatedBlock, emulatedBlockCompressed bytes.Buffer
@@ -170,8 +155,6 @@ func (c *Compression) getHeader() []byte {
 	switch c.CompressionMode {
 	case GzipStore, GzipMin, GzipDefault, GzipMax:
 		return GzipHeader
-	case XZMin, XZDefault:
-		return ExecHeader
 	case LZ4:
 		return LZ4Header
 	case Snappy:
@@ -185,8 +168,6 @@ func (c *Compression) getFooter() []byte {
 	switch c.CompressionMode {
 	case GzipStore, GzipMin, GzipDefault, GzipMax:
 		return []byte{}
-	case XZMin, XZDefault:
-		return []byte{}
 	case LZ4:
 		return LZ4Footer
 	case Snappy:
@@ -207,10 +188,6 @@ func (c *Compression) compressBlock(in []byte, out io.Writer) (compressedSize ui
 		return c.compressBlockGz(in, out, 6)
 	case GzipMax:
 		return c.compressBlockGz(in, out, 9)
-	case XZDefault:
-		return c.compressBlockExec(in, out, c.BinPath, []string{"-c"})
-	case XZMin:
-		return c.compressBlockExec(in, out, c.BinPath, []string{"-c1"})
 	case LZ4:
 		return c.compressBlockLz4(in, out)
 	case Snappy:
@@ -349,10 +326,6 @@ func (d *Decompressor) decompressBlock(in io.Reader, out io.Writer) (n int, err
 	switch d.c.CompressionMode { // Select decompression function based off compression mode
 	case GzipStore, GzipMin, GzipDefault, GzipMax:
 		return decompressBlockRangeGz(in, out)
-	case XZMin:
-		return decompressBlockRangeExec(in, out, d.c.BinPath, []string{"-dc1"})
-	case XZDefault:
-		return decompressBlockRangeExec(in, out, d.c.BinPath, []string{"-dc"})
 	case LZ4:
 		return decompressBlockLz4(in, out, int64(d.c.BlockSize))
 	case Snappy:

backend/press/compression_test.go

@@ -123,11 +123,6 @@ func getCompressibleString(size int) string {
 
 func TestCompression(t *testing.T) {
 	testCases := []string{"lz4", "snappy", "gzip-min"}
-	if checkXZ() {
-		testCases = append(testCases, "xz-min")
-	} else {
-		t.Log("XZ binary not found on current system. Not testing xz.")
-	}
 	for _, tc := range testCases {
 		t.Run(tc, func(t *testing.T) {
 			testSmallLarge(t, tc)

backend/press/press_test.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fstest"
-	"github.com/ncw/rclone/fstest/fstests"
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
@@ -17,10 +17,21 @@ func TestIntegration(t *testing.T) {
 		t.Skip("Skipping as -remote not set")
 	}
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   *fstest.RemoteName,
-		NilObject:                    (*Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{},
+		RemoteName: *fstest.RemoteName,
+		NilObject:  (*Object)(nil),
+		UnimplementableFsMethods: []string{
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"PutUnchecked",
+			"PutStream",
+			"UserInfo",
+			"Disconnect",
+		},
+		UnimplementableObjectMethods: []string{
+			"GetTier",
+			"SetTier",
+		},
 	})
 }
 
@@ -32,10 +43,21 @@ func TestRemoteLz4(t *testing.T) {
 	tempdir := filepath.Join(os.TempDir(), "rclone-press-test-lz4")
 	name := "TestPressLz4"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   name + ":",
-		NilObject:                    (*Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{},
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		UnimplementableFsMethods: []string{
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"PutUnchecked",
+			"PutStream",
+			"UserInfo",
+			"Disconnect",
+		},
+		UnimplementableObjectMethods: []string{
+			"GetTier",
+			"SetTier",
+		},
 		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "type", Value: "press"},
 			{Name: name, Key: "remote", Value: tempdir},
@@ -52,10 +74,21 @@ func TestRemoteGzip(t *testing.T) {
 	tempdir := filepath.Join(os.TempDir(), "rclone-press-test-gzip")
 	name := "TestPressGzip"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   name + ":",
-		NilObject:                    (*Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{},
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		UnimplementableFsMethods: []string{
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"PutUnchecked",
+			"PutStream",
+			"UserInfo",
+			"Disconnect",
+		},
+		UnimplementableObjectMethods: []string{
+			"GetTier",
+			"SetTier",
+		},
 		ExtraConfig: []fstests.ExtraConfigItem{
 			{Name: name, Key: "type", Value: "press"},
 			{Name: name, Key: "remote", Value: tempdir},
@@ -64,25 +97,4 @@ func TestRemoteGzip(t *testing.T) {
 	})
 }
 
-// TestRemoteXZ tests XZ compression
-func TestRemoteXZ(t *testing.T) {
-	if !checkXZ() {
-		t.Skip("XZ binary not found on current system")
-	}
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-press-test-xz")
-	name := "TestPressXZ"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   name + ":",
-		NilObject:                    (*Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
-		UnimplementableObjectMethods: []string{},
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "press"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "compression_mode", Value: "xz-min"},
-		},
-	})
-}
+// TODO: Snappy needs testing