diff --git a/backend/http/http.go b/backend/http/http.go index 0a3cdb7b5..253bc04ab 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -11,6 +11,7 @@ import ( "io" "mime" "net/http" + "net/textproto" "net/url" "path" "strings" @@ -37,6 +38,10 @@ func init() { Description: "HTTP", NewFs: NewFs, CommandHelp: commandHelp, + MetadataInfo: &fs.MetadataInfo{ + System: systemMetadataInfo, + Help: `HTTP metadata keys are case insensitive and are always returned in lower case.`, + }, Options: []fs.Option{{ Name: "url", Help: "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.", @@ -98,6 +103,40 @@ sizes of any files, and some files that don't exist may be in the listing.`, fs.Register(fsi) } +// system metadata keys which this backend owns +var systemMetadataInfo = map[string]fs.MetadataHelp{ + "cache-control": { + Help: "Cache-Control header", + Type: "string", + Example: "no-cache", + }, + "content-disposition": { + Help: "Content-Disposition header", + Type: "string", + Example: "inline", + }, + "content-disposition-filename": { + Help: "Filename retrieved from Content-Disposition header", + Type: "string", + Example: "file.txt", + }, + "content-encoding": { + Help: "Content-Encoding header", + Type: "string", + Example: "gzip", + }, + "content-language": { + Help: "Content-Language header", + Type: "string", + Example: "en-US", + }, + "content-type": { + Help: "Content-Type header", + Type: "string", + Example: "text/plain", + }, +} + // Options defines the configuration for this backend type Options struct { Endpoint string `config:"url"` @@ -126,6 +165,13 @@ type Object struct { size int64 modTime time.Time contentType string + + // Metadata as pointers to strings as they often won't be present + contentDisposition *string // Content-Disposition: header + contentDispositionFilename *string // Filename retrieved from Content-Disposition: header + cacheControl *string // Cache-Control: header + contentEncoding *string // Content-Encoding: header + contentLanguage *string // Content-Language: header } // statusError returns an error if the res contained an error @@ -277,6 +323,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ci: ci, } f.features = (&fs.Features{ + ReadMetadata: true, CanHaveEmptyDirectories: true, }).Fill(ctx, f) @@ -429,6 +476,29 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) { return names, nil } +// parseFilename extracts the filename from a Content-Disposition header +func parseFilename(contentDisposition string) (string, error) { + // Normalize the contentDisposition to canonical MIME format + mediaType, params, err := mime.ParseMediaType(contentDisposition) + if err != nil { + return "", fmt.Errorf("failed to parse contentDisposition: %v", err) + } + + // Check if the contentDisposition is an attachment + if strings.ToLower(mediaType) != "attachment" { + return "", fmt.Errorf("not an attachment: %s", mediaType) + } + + // Extract the filename from the parameters + filename, ok := params["filename"] + if !ok { + return "", fmt.Errorf("filename not found in contentDisposition") + } + + // Decode filename if it contains special encoding + return textproto.TrimString(filename), nil +} + // Adds the configured headers to the request if any func addHeaders(req *http.Request, opt *Options) { for i := 0; i < len(opt.Headers); i += 2 { @@ -577,6 +647,9 @@ func (o *Object) String() string { // Remote the name of the remote HTTP file, relative to the fs root func (o *Object) Remote() string { + if o.contentDispositionFilename != nil { + return *o.contentDispositionFilename + } return o.remote } @@ -634,6 +707,29 @@ func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error { o.modTime = t o.contentType = res.Header.Get("Content-Type") o.size = rest.ParseSizeFromHeaders(res.Header) + contentDisposition := res.Header.Get("Content-Disposition") + if contentDisposition != "" { + o.contentDisposition = &contentDisposition + } + if o.contentDisposition != nil { + var filename string + filename, err = parseFilename(*o.contentDisposition) + if err == nil && filename != "" { + o.contentDispositionFilename = &filename + } + } + cacheControl := res.Header.Get("Cache-Control") + if cacheControl != "" { + o.cacheControl = &cacheControl + } + contentEncoding := res.Header.Get("Content-Encoding") + if contentEncoding != "" { + o.contentEncoding = &contentEncoding + } + contentLanguage := res.Header.Get("Content-Language") + if contentLanguage != "" { + o.contentLanguage = &contentLanguage + } // If NoSlash is set then check ContentType to see if it is a directory if o.fs.opt.NoSlash { @@ -772,6 +868,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str } } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { + metadata = make(fs.Metadata, 6) + if o.contentType != "" { + metadata["content-type"] = o.contentType + } + + // Set system metadata + setMetadata := func(k string, v *string) { + if v == nil || *v == "" { + return + } + metadata[k] = *v + } + setMetadata("content-disposition", o.contentDisposition) + setMetadata("content-disposition-filename", o.contentDispositionFilename) + setMetadata("cache-control", o.cacheControl) + setMetadata("content-language", o.contentLanguage) + setMetadata("content-encoding", o.contentEncoding) + return metadata, nil +} + // Check the interfaces are satisfied var ( _ fs.Fs = &Fs{} @@ -779,4 +899,5 @@ var ( _ fs.Object = &Object{} _ fs.MimeTyper = &Object{} _ fs.Commander = &Fs{} + _ fs.Metadataer = &Object{} ) diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go index 90a2bae78..2bc261eb2 100644 --- a/backend/http/http_internal_test.go +++ b/backend/http/http_internal_test.go @@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple { what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path) assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0]) assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2]) + + // Set the content disposition header for the fifth file + // later we will check if it is set using the metadata method + if r.URL.Path == "/five.txt.gz" { + w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Content-Language", "en-US") + w.Header().Set("Content-Encoding", "gzip") + } + fileServer.ServeHTTP(w, r) }) @@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) { sort.Sort(entries) - require.Equal(t, 4, len(entries)) + require.Equal(t, 5, len(entries)) e := entries[0] - assert.Equal(t, "four", e.Remote()) + assert.Equal(t, "five.txt.gz", e.Remote()) assert.Equal(t, int64(-1), e.Size()) - _, ok := e.(fs.Directory) + _, ok := e.(fs.Object) assert.True(t, ok) e = entries[1] + assert.Equal(t, "four", e.Remote()) + assert.Equal(t, int64(-1), e.Size()) + _, ok = e.(fs.Directory) + assert.True(t, ok) + + e = entries[2] assert.Equal(t, "one%.txt", e.Remote()) assert.Equal(t, int64(5+lineEndSize), e.Size()) _, ok = e.(*Object) assert.True(t, ok) - e = entries[2] + e = entries[3] assert.Equal(t, "three", e.Remote()) assert.Equal(t, int64(-1), e.Size()) _, ok = e.(fs.Directory) assert.True(t, ok) - e = entries[3] + e = entries[4] assert.Equal(t, "two.html", e.Remote()) if noSlash { assert.Equal(t, int64(-1), e.Size()) @@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) { assert.Equal(t, fs.ErrorObjectNotFound, err) } +func TestNewObjectWithMetadata(t *testing.T) { + f := prepare(t) + o, err := f.NewObject(context.Background(), "/five.txt.gz") + require.NoError(t, err) + assert.Equal(t, "five.txt.gz", o.Remote()) + ho, ok := o.(*Object) + assert.True(t, ok) + metadata, err := ho.Metadata(context.Background()) + require.NoError(t, err) + assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"]) + assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"]) + assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"]) + assert.Equal(t, "no-cache", metadata["cache-control"]) + assert.Equal(t, "en-US", metadata["content-language"]) + assert.Equal(t, "gzip", metadata["content-encoding"]) +} + func TestOpen(t *testing.T) { m := prepareServer(t) diff --git a/backend/http/test/files/five.txt.gz b/backend/http/test/files/five.txt.gz new file mode 100644 index 000000000..fa65c0084 Binary files /dev/null and b/backend/http/test/files/five.txt.gz differ diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index fcf3a445b..ab58e3e4c 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -23,7 +23,15 @@ subcommand to specify the protocol, e.g. rclone serve http remote: ` + "```" + ` -Each subcommand has its own options which you can see in their help.`, +When the "--metadata" flag is enabled, the following metadata fields will be provided as headers: +- "content-disposition" +- "cache-control" +- "content-language" +- "content-encoding" +Note: The availability of these fields depends on whether the remote supports metadata. + +Each subcommand has its own options which you can see in their help. +`, Annotations: map[string]string{ "versionIntroduced": "v1.39", }, diff --git a/docs/content/overview.md b/docs/content/overview.md index 68cf44895..28e8db9b4 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -34,7 +34,7 @@ Here is an overview of the major features of each cloud storage system. | Google Photos | - | - | No | Yes | R | - | | HDFS | - | R/W | No | No | - | - | | HiDrive | HiDrive ¹² | R/W | No | No | - | - | -| HTTP | - | R | No | No | R | - | +| HTTP | - | R | No | No | R | R | | iCloud Drive | - | R | No | No | - | - | | Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU | | Jottacloud | MD5 | R/W | Yes | No | R | RW | diff --git a/fs/object/object.go b/fs/object/object.go index 9b4e25afe..07dd8a320 100644 --- a/fs/object/object.go +++ b/fs/object/object.go @@ -191,11 +191,12 @@ var _ fs.Fs = MemoryFs // MemoryObject is an in memory object type MemoryObject struct { - remote string - modTime time.Time - content []byte - meta fs.Metadata - fs fs.Fs + remote string + modTime time.Time + content []byte + meta fs.Metadata + fs fs.Fs + mimeType string } // NewMemoryObject returns an in memory Object with the modTime and content passed in @@ -214,6 +215,12 @@ func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject { return o } +// WithMimeType adds mimeType to the MemoryObject +func (o *MemoryObject) WithMimeType(mimeType string) *MemoryObject { + o.mimeType = mimeType + return o +} + // Content returns the underlying buffer func (o *MemoryObject) Content() []byte { return o.content @@ -329,8 +336,14 @@ func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) { return o.meta, nil } +// MimeType on the object +func (o *MemoryObject) MimeType(ctx context.Context) string { + return o.mimeType +} + // Check interfaces var ( _ fs.Object = (*MemoryObject)(nil) + _ fs.MimeTyper = (*MemoryObject)(nil) _ fs.Metadataer = (*MemoryObject)(nil) ) diff --git a/fs/object/object_test.go b/fs/object/object_test.go index be90c771e..d1597acc3 100644 --- a/fs/object/object_test.go +++ b/fs/object/object_test.go @@ -87,6 +87,7 @@ func TestMemoryObject(t *testing.T) { content = content[:6] // make some extra cap o := object.NewMemoryObject(remote, now, content) + o.WithMimeType("text/plain; charset=utf-8") assert.Equal(t, content, o.Content()) assert.Equal(t, object.MemoryFs, o.Fs()) @@ -95,6 +96,7 @@ func TestMemoryObject(t *testing.T) { assert.Equal(t, now, o.ModTime(context.Background())) assert.Equal(t, int64(len(content)), o.Size()) assert.Equal(t, true, o.Storable()) + assert.Equal(t, "text/plain; charset=utf-8", o.MimeType(context.Background())) Hash, err := o.Hash(context.Background(), hash.MD5) assert.NoError(t, err) diff --git a/lib/http/serve/serve.go b/lib/http/serve/serve.go index 21f3c3322..244509b47 100644 --- a/lib/http/serve/serve.go +++ b/lib/http/serve/serve.go @@ -39,6 +39,26 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) { modTime := o.ModTime(r.Context()) w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat)) + // Set metadata headers if present + metadata, err := fs.GetMetadata(r.Context(), o) + if err != nil { + fs.Debugf(o, "Request get metadata error: %v", err) + } + if metadata != nil { + if metadata["content-disposition"] != "" { + w.Header().Set("Content-Disposition", metadata["content-disposition"]) + } + if metadata["cache-control"] != "" { + w.Header().Set("Cache-Control", metadata["cache-control"]) + } + if metadata["content-language"] != "" { + w.Header().Set("Content-Language", metadata["content-language"]) + } + if metadata["content-encoding"] != "" { + w.Header().Set("Content-Encoding", metadata["content-encoding"]) + } + } + if r.Method == "HEAD" { return } diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go index 79daf773e..ed716cad9 100644 --- a/lib/http/serve/serve_test.go +++ b/lib/http/serve/serve_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fstest/mockobject" "github.com/stretchr/testify/assert" ) @@ -82,3 +84,23 @@ func TestObjectBadRange(t *testing.T) { body, _ := io.ReadAll(resp.Body) assert.Equal(t, "Bad Request\n", string(body)) } + +func TestObjectHEADMetadata(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil) + m := fs.Metadata{ + "content-disposition": "inline", + "cache-control": "no-cache", + "content-language": "en", + "content-encoding": "gzip", + } + o := object.NewMemoryObject("aFile", time.Now(), []byte("")). + WithMetadata(m).WithMimeType("text/plain; charset=utf-8") + Object(w, r, o) + resp := w.Result() + assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) + assert.Equal(t, "inline", resp.Header.Get("Content-Disposition")) + assert.Equal(t, "no-cache", resp.Header.Get("Cache-Control")) + assert.Equal(t, "en", resp.Header.Get("Content-Language")) + assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding")) +}