diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index 0acc5149f..e45c0ab85 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -41,9 +41,10 @@ var OptionsInfo = fs.Options{}. // Options required for http server type Options struct { - Auth libhttp.AuthConfig - HTTP libhttp.Config - Template libhttp.TemplateConfig + Auth libhttp.AuthConfig + HTTP libhttp.Config + Template libhttp.TemplateConfig + DisableZip bool } // DefaultOpt is the default values used for Options @@ -69,6 +70,7 @@ func init() { flags.AddFlagsFromOptions(flagSet, "", OptionsInfo) vfsflags.AddFlags(flagSet) proxyflags.AddFlags(flagSet) + flagSet.BoolVar(&Opt.DisableZip, "disable-zip", false, "Disable zip download of directories") cmdserve.Command.AddCommand(Command) cmdserve.AddRc("http", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) { // Read VFS Opts @@ -257,6 +259,24 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string return } dir := node.(*vfs.Dir) + + if r.URL.Query().Get("download") == "zip" && !s.opt.DisableZip { + fs.Infof(dirRemote, "%s: Zipping directory", r.RemoteAddr) + zipName := path.Base(dirRemote) + if dirRemote == "" { + zipName = "root" + } + w.Header().Set("Content-Disposition", "attachment; filename=\""+zipName+".zip\"") + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) + err := vfs.CreateZip(ctx, dir, w) + if err != nil { + serve.Error(ctx, dirRemote, w, "Failed to create zip", err) + return + } + return + } + dirEntries, err := dir.ReadDirAll() if err != nil { serve.Error(ctx, dirRemote, w, "Failed to list directory", err) @@ -280,6 +300,8 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string // Set the Last-Modified header to the timestamp w.Header().Set("Last-Modified", dir.ModTime().UTC().Format(http.TimeFormat)) + directory.DisableZip = s.opt.DisableZip + directory.Serve(w, r) } diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go index 3044c141a..5d8ccdc06 100644 --- a/cmd/serve/http/http_test.go +++ b/cmd/serve/http/http_test.go @@ -4,6 +4,7 @@ import ( "context" "flag" "io" + stdfs "io/fs" "net/http" "os" "path/filepath" @@ -75,6 +76,16 @@ func start(ctx context.Context, t *testing.T, f fs.Fs) (s *HTTP, testURL string) return s, testURL } +// setAllModTimes walks root and sets atime/mtime to t for every file & directory. +func setAllModTimes(root string, t time.Time) error { + return filepath.WalkDir(root, func(path string, d stdfs.DirEntry, err error) error { + if err != nil { + return err + } + return os.Chtimes(path, t, t) + }) +} + var ( datedObject = "two.txt" expectedTime = time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC) @@ -123,6 +134,8 @@ func testGET(t *testing.T, useProxy bool) { f = nil } else { + // set all the mod times to expectedTime + require.NoError(t, setAllModTimes("testdata/files", expectedTime)) // Create a test Fs var err error f, err = fs.NewFs(context.Background(), "testdata/files") @@ -233,6 +246,16 @@ func testGET(t *testing.T, useProxy bool) { Range: "bytes=3-", Golden: "testdata/golden/two3-.txt", }, + { + URL: "/?download=zip", + Status: http.StatusOK, + Golden: "testdata/golden/root.zip", + }, + { + URL: "/three/?download=zip", + Status: http.StatusOK, + Golden: "testdata/golden/three.zip", + }, } { method := test.Method if method == "" { diff --git a/cmd/serve/http/testdata/golden/root.zip b/cmd/serve/http/testdata/golden/root.zip new file mode 100644 index 000000000..a3a710df4 Binary files /dev/null and b/cmd/serve/http/testdata/golden/root.zip differ diff --git a/cmd/serve/http/testdata/golden/three.zip b/cmd/serve/http/testdata/golden/three.zip new file mode 100644 index 000000000..bcffe9819 Binary files /dev/null and b/cmd/serve/http/testdata/golden/three.zip differ diff --git a/lib/http/serve/dir.go b/lib/http/serve/dir.go index ae1ead6a0..fd53256b3 100644 --- a/lib/http/serve/dir.go +++ b/lib/http/serve/dir.go @@ -21,6 +21,7 @@ import ( type DirEntry struct { remote string URL string + ZipURL string Leaf string IsDir bool Size int64 @@ -32,6 +33,8 @@ type Directory struct { DirRemote string Title string Name string + ZipURL string + DisableZip bool Entries []DirEntry Query string HTMLTemplate *template.Template @@ -70,6 +73,7 @@ func NewDirectory(dirRemote string, htmlTemplate *template.Template) *Directory DirRemote: dirRemote, Title: fmt.Sprintf("Directory listing of /%s", dirRemote), Name: fmt.Sprintf("/%s", dirRemote), + ZipURL: "?download=zip", HTMLTemplate: htmlTemplate, Breadcrumb: breadcrumb, } @@ -99,11 +103,15 @@ func (d *Directory) AddHTMLEntry(remote string, isDir bool, size int64, modTime d.Entries = append(d.Entries, DirEntry{ remote: remote, URL: rest.URLPathEscape(urlRemote) + d.Query, + ZipURL: "", Leaf: leaf, IsDir: isDir, Size: size, ModTime: modTime, }) + if isDir { + d.Entries[len(d.Entries)-1].ZipURL = rest.URLPathEscape(urlRemote) + "?download=zip" + } } // AddEntry adds an entry to that directory diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go index 19ab22f57..3fd146724 100644 --- a/lib/http/serve/dir_test.go +++ b/lib/http/serve/dir_test.go @@ -46,11 +46,11 @@ func TestAddHTMLEntry(t *testing.T) { d.AddHTMLEntry("a/b/c/colon:colon.txt", false, 64, modtime) d.AddHTMLEntry("\"quotes\".txt", false, 64, modtime) assert.Equal(t, []DirEntry{ - {remote: "", URL: "/", Leaf: "/", IsDir: true, Size: 0, ModTime: modtime}, - {remote: "dir", URL: "dir/", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime}, - {remote: "a/b/c/d.txt", URL: "d.txt", Leaf: "d.txt", IsDir: false, Size: 64, ModTime: modtime}, - {remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", Leaf: "colon:colon.txt", IsDir: false, Size: 64, ModTime: modtime}, - {remote: "\"quotes\".txt", URL: "%22quotes%22.txt", Leaf: "\"quotes\".txt", Size: 64, IsDir: false, ModTime: modtime}, + {remote: "", URL: "/", ZipURL: "/?download=zip", Leaf: "/", IsDir: true, Size: 0, ModTime: modtime}, + {remote: "dir", URL: "dir/", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime}, + {remote: "a/b/c/d.txt", URL: "d.txt", ZipURL: "", Leaf: "d.txt", IsDir: false, Size: 64, ModTime: modtime}, + {remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", ZipURL: "", Leaf: "colon:colon.txt", IsDir: false, Size: 64, ModTime: modtime}, + {remote: "\"quotes\".txt", URL: "%22quotes%22.txt", ZipURL: "", Leaf: "\"quotes\".txt", Size: 64, IsDir: false, ModTime: modtime}, }, d.Entries) // Now test with a query parameter @@ -58,8 +58,8 @@ func TestAddHTMLEntry(t *testing.T) { d.AddHTMLEntry("file", false, 64, modtime) d.AddHTMLEntry("dir", true, 0, modtime) assert.Equal(t, []DirEntry{ - {remote: "file", URL: "file?potato=42", Leaf: "file", IsDir: false, Size: 64, ModTime: modtime}, - {remote: "dir", URL: "dir/?potato=42", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime}, + {remote: "file", URL: "file?potato=42", ZipURL: "", Leaf: "file", IsDir: false, Size: 64, ModTime: modtime}, + {remote: "dir", URL: "dir/?potato=42", ZipURL: "dir/?download=zip", Leaf: "dir/", IsDir: true, Size: 0, ModTime: modtime}, }, d.Entries) } diff --git a/lib/http/server.go b/lib/http/server.go index 25e11f269..83d0162dd 100644 --- a/lib/http/server.go +++ b/lib/http/server.go @@ -59,6 +59,8 @@ inserts leading and trailing "/" on ` + "`--{{ .Prefix }}baseurl`" + `, so ` + " ` + "`--{{ .Prefix }}baseurl \"/rclone\"` and `--{{ .Prefix }}baseurl \"/rclone/\"`" + ` are all treated identically. +` + "`--{{ .Prefix }}disable-zip`" + ` may be set to disable the zipping download option. + #### TLS (SSL) By default this will serve over http. If you want you can serve over diff --git a/lib/http/templates/index.html b/lib/http/templates/index.html index 348050c02..cec58f1d7 100644 --- a/lib/http/templates/index.html +++ b/lib/http/templates/index.html @@ -21,7 +21,7 @@ Modifications: Adapted to rclone markup --> - @@ -206,6 +219,9 @@ footer { + + + @@ -233,6 +249,15 @@ footer {

{{range $i, $crumb := .Breadcrumb}}{{html $crumb.Text}}{{if ne $i 0}}/{{end}}{{end}} + + {{- if not .DisableZip}} + + + + + + {{- end}} +

@@ -283,6 +308,13 @@ footer { {{- end}} {{html .Leaf}} + {{- if and .IsDir (not $.DisableZip)}} + + + + + + {{- end}} {{- if .IsDir}} — diff --git a/vfs/zip.go b/vfs/zip.go new file mode 100644 index 000000000..287777bed --- /dev/null +++ b/vfs/zip.go @@ -0,0 +1,73 @@ +package vfs + +import ( + "archive/zip" + "context" + "fmt" + "io" + "os" + + "github.com/rclone/rclone/fs" +) + +// CreateZip creates a zip file from a vfs.Dir writing it to w +func CreateZip(ctx context.Context, dir *Dir, w io.Writer) (err error) { + zipWriter := zip.NewWriter(w) + defer fs.CheckClose(zipWriter, &err) + var walk func(dir *Dir, root string) error + walk = func(dir *Dir, root string) error { + nodes, err := dir.ReadDirAll() + if err != nil { + return fmt.Errorf("create zip directory read: %w", err) + } + for _, node := range nodes { + switch e := node.(type) { + case *File: + in, err := e.Open(os.O_RDONLY) + if err != nil { + return fmt.Errorf("create zip open file: %w", err) + } + header := &zip.FileHeader{ + Name: root + e.Name(), + Method: zip.Deflate, + Modified: e.ModTime(), + } + fileWriter, err := zipWriter.CreateHeader(header) + if err != nil { + fs.CheckClose(in, &err) + return fmt.Errorf("create zip file header: %w", err) + } + _, err = io.Copy(fileWriter, in) + if err != nil { + fs.CheckClose(in, &err) + return fmt.Errorf("create zip copy: %w", err) + } + fs.CheckClose(in, &err) + case *Dir: + name := root + e.Path() + if name != "" && name[len(name)-1] != '/' { + name += "/" + } + header := &zip.FileHeader{ + Name: name, + Method: zip.Store, + Modified: e.ModTime(), + } + _, err := zipWriter.CreateHeader(header) + if err != nil { + return fmt.Errorf("create zip directory header: %w", err) + } + err = walk(e, name) + if err != nil { + return err + } + } + } + return nil + } + err = walk(dir, "") + if err != nil { + return err + } + return nil +} diff --git a/vfs/zip_test.go b/vfs/zip_test.go new file mode 100644 index 000000000..cf8207ca6 --- /dev/null +++ b/vfs/zip_test.go @@ -0,0 +1,156 @@ +package vfs + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "strings" + "testing" + + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/require" +) + +func readZip(t *testing.T, buf *bytes.Buffer) *zip.Reader { + t.Helper() + r, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.NoError(t, err) + return r +} + +func mustCreateZip(t *testing.T, d *Dir) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + require.NoError(t, CreateZip(context.Background(), d, &buf)) + return &buf +} + +func zipReadFile(t *testing.T, zr *zip.Reader, match func(name string) bool) ([]byte, string) { + t.Helper() + for _, f := range zr.File { + if strings.HasSuffix(f.Name, "/") { + continue + } + if match(f.Name) { + rc, err := f.Open() + require.NoError(t, err) + defer func() { require.NoError(t, rc.Close()) }() + b, err := io.ReadAll(rc) + require.NoError(t, err) + return b, f.Name + } + } + t.Fatalf("zip entry matching predicate not found") + return nil, "" +} + +func TestZipManyFiles(t *testing.T) { + r, vfs := newTestVFS(t) + + const N = 5 + want := make(map[string]string, N) + items := make([]fstest.Item, 0, N) + + for i := range N { + name := fmt.Sprintf("flat/f%03d.txt", i) + data := strings.Repeat(fmt.Sprintf("line-%d\n", i), (i%5)+1) + it := r.WriteObject(context.Background(), name, data, t1) + items = append(items, it) + want[name[strings.LastIndex(name, "/")+1:]] = data + } + r.CheckRemoteItems(t, items...) + + node, err := vfs.Stat("flat") + require.NoError(t, err) + dir := node.(*Dir) + + buf := mustCreateZip(t, dir) + zr := readZip(t, buf) + + // count only file entries (skip dir entries with trailing "/") + files := 0 + for _, f := range zr.File { + if !strings.HasSuffix(f.Name, "/") { + files++ + } + } + require.Equal(t, N, files) + + // validate contents by base name + for base, data := range want { + got, _ := zipReadFile(t, zr, func(name string) bool { return name == base }) + require.Equal(t, data, string(got), "mismatch for %s", base) + } +} + +func TestZipManySubDirs(t *testing.T) { + r, vfs := newTestVFS(t) + + r.WriteObject(context.Background(), "a/top.txt", "top", t1) + r.WriteObject(context.Background(), "a/b/mid.txt", "mid", t1) + r.WriteObject(context.Background(), "a/b/c/deep.txt", "deep", t1) + + node, err := vfs.Stat("a") + require.NoError(t, err) + dir := node.(*Dir) + + buf := mustCreateZip(t, dir) + zr := readZip(t, buf) + + // paths may include directory prefixes; assert by suffix + got, name := zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/top.txt") || n == "top.txt" }) + require.Equal(t, "top", string(got), "bad content for %s", name) + + got, name = zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/mid.txt") || n == "mid.txt" }) + require.Equal(t, "mid", string(got), "bad content for %s", name) + + got, name = zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/deep.txt") || n == "deep.txt" }) + require.Equal(t, "deep", string(got), "bad content for %s", name) +} + +func TestZipLargeFiles(t *testing.T) { + r, vfs := newTestVFS(t) + + data := random.String(5 * 1024 * 1024) + sum := sha256.Sum256([]byte(data)) + + r.WriteObject(context.Background(), "bigdir/big.bin", data, t1) + + node, err := vfs.Stat("bigdir") + require.NoError(t, err) + dir := node.(*Dir) + + buf := mustCreateZip(t, dir) + zr := readZip(t, buf) + + got, _ := zipReadFile(t, zr, func(n string) bool { return n == "big.bin" || strings.HasSuffix(n, "/big.bin") }) + require.Equal(t, sum, sha256.Sum256(got)) +} + +func TestZipDirsInRoot(t *testing.T) { + r, vfs := newTestVFS(t) + + r.WriteObject(context.Background(), "dir1/a.txt", "x", t1) + r.WriteObject(context.Background(), "dir2/b.txt", "y", t1) + r.WriteObject(context.Background(), "dir3/c.txt", "z", t1) + + root, err := vfs.Root() + require.NoError(t, err) + + buf := mustCreateZip(t, root) + zr := readZip(t, buf) + + // Check each file exists (ignore exact directory-entry names) + gx, _ := zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/a.txt") }) + require.Equal(t, "x", string(gx)) + + gy, _ := zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/b.txt") }) + require.Equal(t, "y", string(gy)) + + gz, _ := zipReadFile(t, zr, func(n string) bool { return strings.HasSuffix(n, "/c.txt") }) + require.Equal(t, "z", string(gz)) +}