drive: prevent retries when query filter returned no entries

https://forum.rclone.org/t/performance-degradation-between-v1-56-2-and-v1-57-0-when-copying-to-google-drive-using-max-age-min-age-and-fast-list/27580/2

backend/chunker/chunker_test.go

@@ -44,7 +44,6 @@ func TestIntegration(t *testing.T) {
 			"UserInfo",
 			"Disconnect",
 		},
-		QuickTestOK: true,
 	}
 	if *fstest.RemoteName == "" {
 		name := "TestChunker"

backend/compress/compress_test.go

@@ -16,9 +16,6 @@ import (
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
-	if *fstest.RemoteName == "" {
-		t.Skip("Skipping as -remote not set")
-	}
 	opt := fstests.Opt{
 		RemoteName: *fstest.RemoteName,
 		NilObject:  (*Object)(nil),
@@ -64,6 +61,5 @@ func TestRemoteGzip(t *testing.T) {
 			{Name: name, Key: "remote", Value: tempdir},
 			{Name: name, Key: "compression_mode", Value: "gzip"},
 		},
-		QuickTestOK: true,
 	})
 }

backend/crypt/crypt.go

@@ -1049,10 +1049,6 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	}
 	// if this is wrapping a local object then we work out the hash
 	if srcObj.Fs().Features().IsLocal {
-		if o.f.opt.NoDataEncryption {
-			// If no encryption, just return the hash of the underlying object
-			return srcObj.Hash(ctx, hash)
-		}
 		// Read the data and encrypt it to calculate the hash
 		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
 		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)

backend/crypt/crypt_internal_test.go

@@ -77,11 +77,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	enc, err := f.cipher.newEncrypter(inBuf, nil)
 	require.NoError(t, err)
 	nonce := enc.nonce // read the nonce at the start
-	if f.opt.NoDataEncryption {
-		_, err = outBuf.WriteString(contents)
-	} else {
-		_, err = io.Copy(&outBuf, enc)
-	}
+	_, err = io.Copy(&outBuf, enc)
 	require.NoError(t, err)
 
 	var oi fs.ObjectInfo = obj
@@ -100,12 +96,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	assert.NotEqual(t, path, src.Remote())
 
 	// Test ObjectInfo.Hash
-	var wantHash [md5.Size]byte
-	if f.opt.NoDataEncryption {
-		wantHash = md5.Sum([]byte(contents))
-	} else {
-		wantHash = md5.Sum(outBuf.Bytes())
-	}
+	wantHash := md5.Sum(outBuf.Bytes())
 	gotHash, err := src.Hash(ctx, hash.MD5)
 	require.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)

backend/crypt/crypt_test.go

@@ -46,7 +46,6 @@ func TestStandardBase32(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -68,7 +67,6 @@ func TestStandardBase64(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -90,7 +88,6 @@ func TestStandardBase32768(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -112,7 +109,6 @@ func TestOff(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -135,7 +131,6 @@ func TestObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -159,6 +154,5 @@ func TestNoDataObfuscate(t *testing.T) {
 		SkipBadWindowsCharacters:     true,
 		UnimplementableFsMethods:     []string{"OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }

backend/drive/drive.go

@@ -1693,6 +1693,11 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 	var paths []string
 	var grouping int32
 
+	usingQueryFilter := false
+	if fi, use := filter.GetConfig(ctx), filter.GetUseFilter(ctx); fi != nil && use {
+		usingQueryFilter = true
+	}
+
 	for dir := range in {
 		dirs = append(dirs[:0], dir.id)
 		paths = append(paths[:0], dir.path)
@@ -1765,7 +1770,8 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// drive where (A in parents) or (B in parents) returns nothing
 		// sometimes. See #3114, #4289 and
 		// https://issuetracker.google.com/issues/149522397
-		if len(dirs) > 1 && !foundItems {
+		// However, empty result is legitimate if query filter was applied.
+		if len(dirs) > 1 && !foundItems && !usingQueryFilter {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
 				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
@@ -1783,7 +1789,8 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		}
 		// If using a grouping of 1 and dir was empty then check to see if it
 		// is part of the group that caused grouping to be disabled.
-		if grouping == 1 && len(dirs) == 1 && !foundItems {
+		// However, empty result is legitimate if query filter was applied.
+		if grouping == 1 && len(dirs) == 1 && !foundItems && !usingQueryFilter {
 			f.listRmu.Lock()
 			if _, found := f.listRempties[dirs[0]]; found {
 				// Remove the ID

backend/hasher/hasher_test.go

@@ -25,7 +25,6 @@ func TestIntegration(t *testing.T) {
 			"OpenWriterAt",
 		},
 		UnimplementableObjectMethods: []string{},
-		QuickTestOK:                  true,
 	}
 	if *fstest.RemoteName == "" {
 		tempDir := filepath.Join(os.TempDir(), "rclone-hasher-test")

backend/local/local_test.go

@@ -11,8 +11,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "",
-		NilObject:   (*local.Object)(nil),
-		QuickTestOK: true,
+		RemoteName: "",
+		NilObject:  (*local.Object)(nil),
 	})
 }

backend/memory/memory_test.go

@@ -10,8 +10,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  ":memory:",
-		NilObject:   (*Object)(nil),
-		QuickTestOK: true,
+		RemoteName: ":memory:",
+		NilObject:  (*Object)(nil),
 	})
 }

backend/union/union_test.go

@@ -42,7 +42,6 @@ func TestStandard(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -65,7 +64,6 @@ func TestRO(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -88,7 +86,6 @@ func TestNC(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -111,7 +108,6 @@ func TestPolicy1(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -134,7 +130,6 @@ func TestPolicy2(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -157,6 +152,5 @@ func TestPolicy3(t *testing.T) {
 		},
 		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
 		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }

fs/fshttp/http.go

@@ -136,12 +136,14 @@ func NewClient(ctx context.Context) *http.Client {
 // Transport is our http Transport which wraps an http.Transport
 // * Sets the User Agent
 // * Does logging
+// * Updates metrics
 type Transport struct {
 	*http.Transport
 	dump          fs.DumpFlags
 	filterRequest func(req *http.Request)
 	userAgent     string
 	headers       []*fs.HTTPOption
+	metrics       *Metrics
 }
 
 // newTransport wraps the http.Transport passed in and logs all
@@ -152,6 +154,7 @@ func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
 		dump:      ci.Dump,
 		userAgent: ci.UserAgent,
 		headers:   ci.Headers,
+		metrics:   DefaultMetrics,
 	}
 }
 
@@ -283,6 +286,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorResp)
 		logMutex.Unlock()
 	}
+	// Update metrics
+	t.metrics.onResponse(req, resp)
+
 	if err == nil {
 		checkServerTime(req, resp)
 	}

fs/fshttp/prometheus.go

@@ -0,0 +1,51 @@
+package fshttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics provide Transport HTTP level metrics.
+type Metrics struct {
+	StatusCode *prometheus.CounterVec
+}
+
+// NewMetrics creates a new metrics instance, the instance shall be assigned to
+// DefaultMetrics before any processing takes place.
+func NewMetrics(namespace string) *Metrics {
+	return &Metrics{
+		StatusCode: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: "http",
+			Name:      "status_code",
+		}, []string{"host", "method", "code"}),
+	}
+}
+
+// DefaultMetrics specifies metrics used for new Transports.
+var DefaultMetrics = (*Metrics)(nil)
+
+// Collectors returns all prometheus metrics as collectors for registration.
+func (m *Metrics) Collectors() []prometheus.Collector {
+	if m == nil {
+		return nil
+	}
+	return []prometheus.Collector{
+		m.StatusCode,
+	}
+}
+
+func (m *Metrics) onResponse(req *http.Request, resp *http.Response) {
+	if m == nil {
+		return
+	}
+
+	var statusCode = 0
+	if resp != nil {
+		statusCode = resp.StatusCode
+	}
+
+	m.StatusCode.WithLabelValues(req.Host, req.Method, fmt.Sprint(statusCode)).Inc()
+}

fs/rc/rcserver/rcserver.go

@@ -18,23 +18,22 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs/rc/webgui"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/skratchdot/open-golang/open"
-
 	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fs/rc/jobs"
 	"github.com/rclone/rclone/fs/rc/rcflags"
+	"github.com/rclone/rclone/fs/rc/webgui"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/skratchdot/open-golang/open"
 )
 
 var promHandler http.Handler
@@ -43,6 +42,13 @@ var onlyOnceWarningAllowOrigin sync.Once
 func init() {
 	rcloneCollector := accounting.NewRcloneCollector(context.Background())
 	prometheus.MustRegister(rcloneCollector)
+
+	m := fshttp.NewMetrics("rclone")
+	for _, c := range m.Collectors() {
+		prometheus.MustRegister(c)
+	}
+	fshttp.DefaultMetrics = m
+
 	promHandler = promhttp.Handler()
 }
 

fstest/fstests/fstests.go

@@ -295,7 +295,6 @@ type Opt struct {
 	SkipFsCheckWrap              bool     // if set skip FsCheckWrap
 	SkipObjectCheckWrap          bool     // if set skip ObjectCheckWrap
 	SkipInvalidUTF8              bool     // if set skip invalid UTF-8 checks
-	QuickTestOK                  bool     // OK to run this test in "make quicktest"
 }
 
 // returns true if x is found in ss
@@ -360,7 +359,7 @@ func Run(t *testing.T, opt *Opt) {
 		unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
 	)
 
-	if !opt.QuickTestOK && strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
+	if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
 		t.Skip("quicktest only")
 	}