serve http: add gzip compression

Add gzip compression for directory listings and text assets served over HTTP.

This reduces the rclone repository file listing from 40 kB to 8 kB and reduces
the rclone MANUAL.txt from 2.7 MB to 700 kB.

This makes listings and assets served across the network load faster.

The compression level of 5 should be a good balance between size and speed.

cmd/serve/http/http.go

@@ -202,6 +202,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 
 	router := s.server.Router()
 	router.Use(
+		middleware.Compress(5),
 		middleware.SetHeader("Accept-Ranges", "bytes"),
 		middleware.SetHeader("Server", "rclone/"+fs.Version),
 	)

cmd/serve/http/http_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"compress/gzip"
 	"context"
 	"flag"
 	"io"
@@ -347,6 +348,66 @@ func TestFavicon(t *testing.T) {
 	})
 }
 
+func TestCompressedDirectoryListing(t *testing.T) {
+	ctx := context.Background()
+	require.NoError(t, setAllModTimes("testdata/files", expectedTime))
+	f, err := fs.NewFs(ctx, "testdata/files")
+	require.NoError(t, err)
+
+	s, testURL := start(ctx, t, f)
+	defer func() { assert.NoError(t, s.server.Shutdown()) }()
+
+	req, err := http.NewRequest("GET", testURL, nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	req.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+	gr, err := gzip.NewReader(resp.Body)
+	require.NoError(t, err)
+	defer func() { _ = gr.Close() }()
+
+	body, err := io.ReadAll(gr)
+	require.NoError(t, err)
+	assert.Contains(t, string(body), "Directory listing of /")
+}
+
+func TestCompressedTextFile(t *testing.T) {
+	ctx := context.Background()
+	require.NoError(t, setAllModTimes("testdata/files", expectedTime))
+	f, err := fs.NewFs(ctx, "testdata/files")
+	require.NoError(t, err)
+
+	s, testURL := start(ctx, t, f)
+	defer func() { assert.NoError(t, s.server.Shutdown()) }()
+
+	req, err := http.NewRequest("GET", testURL+"two.txt", nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	req.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+	gr, err := gzip.NewReader(resp.Body)
+	require.NoError(t, err)
+	defer func() { _ = gr.Close() }()
+
+	body, err := io.ReadAll(gr)
+	require.NoError(t, err)
+	assert.Equal(t, "0123456789\n", string(body))
+}
+
 func TestRc(t *testing.T) {
 	servetest.TestRc(t, rc.Params{
 		"type":           "http",

lib/http/serve/dir.go

@@ -245,6 +245,7 @@ func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
 		Error(ctx, d.DirRemote, w, "Failed to render template", err)
 		return
 	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	w.Header().Set("Content-Length", fmt.Sprintf("%d", buf.Len()))
 	_, err = buf.WriteTo(w)
 	if err != nil {