http: add basic metadata and provide it via serve

Co-authored-by: dougal <147946567+roucc@users.noreply.github.com>

backend/http/http.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"path"
 	"strings"
@@ -37,6 +38,10 @@ func init() {
 		Description: "HTTP",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
+		MetadataInfo: &fs.MetadataInfo{
+			System: systemMetadataInfo,
+			Help:   `HTTP metadata keys are case insensitive and are always returned in lower case.`,
+		},
 		Options: []fs.Option{{
 			Name:     "url",
 			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
@@ -98,6 +103,40 @@ sizes of any files, and some files that don't exist may be in the listing.`,
 	fs.Register(fsi)
 }
 
+// system metadata keys which this backend owns
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+	"cache-control": {
+		Help:    "Cache-Control header",
+		Type:    "string",
+		Example: "no-cache",
+	},
+	"content-disposition": {
+		Help:    "Content-Disposition header",
+		Type:    "string",
+		Example: "inline",
+	},
+	"content-disposition-filename": {
+		Help:    "Filename retrieved from Content-Disposition header",
+		Type:    "string",
+		Example: "file.txt",
+	},
+	"content-encoding": {
+		Help:    "Content-Encoding header",
+		Type:    "string",
+		Example: "gzip",
+	},
+	"content-language": {
+		Help:    "Content-Language header",
+		Type:    "string",
+		Example: "en-US",
+	},
+	"content-type": {
+		Help:    "Content-Type header",
+		Type:    "string",
+		Example: "text/plain",
+	},
+}
+
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string          `config:"url"`
@@ -126,6 +165,13 @@ type Object struct {
 	size        int64
 	modTime     time.Time
 	contentType string
+
+	// Metadata as pointers to strings as they often won't be present
+	contentDisposition         *string // Content-Disposition: header
+	contentDispositionFilename *string // Filename retrieved from Content-Disposition: header
+	cacheControl               *string // Cache-Control: header
+	contentEncoding            *string // Content-Encoding: header
+	contentLanguage            *string // Content-Language: header
 }
 
 // statusError returns an error if the res contained an error
@@ -277,6 +323,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:   ci,
 	}
 	f.features = (&fs.Features{
+		ReadMetadata:            true,
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
 
@@ -429,6 +476,29 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	return names, nil
 }
 
+// parseFilename extracts the filename from a Content-Disposition header
+func parseFilename(contentDisposition string) (string, error) {
+	// Normalize the contentDisposition to canonical MIME format
+	mediaType, params, err := mime.ParseMediaType(contentDisposition)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse contentDisposition: %v", err)
+	}
+
+	// Check if the contentDisposition is an attachment
+	if strings.ToLower(mediaType) != "attachment" {
+		return "", fmt.Errorf("not an attachment: %s", mediaType)
+	}
+
+	// Extract the filename from the parameters
+	filename, ok := params["filename"]
+	if !ok {
+		return "", fmt.Errorf("filename not found in contentDisposition")
+	}
+
+	// Decode filename if it contains special encoding
+	return textproto.TrimString(filename), nil
+}
+
 // Adds the configured headers to the request if any
 func addHeaders(req *http.Request, opt *Options) {
 	for i := 0; i < len(opt.Headers); i += 2 {
@@ -577,6 +647,9 @@ func (o *Object) String() string {
 
 // Remote the name of the remote HTTP file, relative to the fs root
 func (o *Object) Remote() string {
+	if o.contentDispositionFilename != nil {
+		return *o.contentDispositionFilename
+	}
 	return o.remote
 }
 
@@ -634,6 +707,29 @@ func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
 	o.size = rest.ParseSizeFromHeaders(res.Header)
+	contentDisposition := res.Header.Get("Content-Disposition")
+	if contentDisposition != "" {
+		o.contentDisposition = &contentDisposition
+	}
+	if o.contentDisposition != nil {
+		var filename string
+		filename, err = parseFilename(*o.contentDisposition)
+		if err == nil && filename != "" {
+			o.contentDispositionFilename = &filename
+		}
+	}
+	cacheControl := res.Header.Get("Cache-Control")
+	if cacheControl != "" {
+		o.cacheControl = &cacheControl
+	}
+	contentEncoding := res.Header.Get("Content-Encoding")
+	if contentEncoding != "" {
+		o.contentEncoding = &contentEncoding
+	}
+	contentLanguage := res.Header.Get("Content-Language")
+	if contentLanguage != "" {
+		o.contentLanguage = &contentLanguage
+	}
 
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
@@ -772,6 +868,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	}
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	metadata = make(fs.Metadata, 6)
+	if o.contentType != "" {
+		metadata["content-type"] = o.contentType
+	}
+
+	// Set system metadata
+	setMetadata := func(k string, v *string) {
+		if v == nil || *v == "" {
+			return
+		}
+		metadata[k] = *v
+	}
+	setMetadata("content-disposition", o.contentDisposition)
+	setMetadata("content-disposition-filename", o.contentDispositionFilename)
+	setMetadata("cache-control", o.cacheControl)
+	setMetadata("content-language", o.contentLanguage)
+	setMetadata("content-encoding", o.contentEncoding)
+	return metadata, nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
@@ -779,4 +899,5 @@ var (
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
 	_ fs.Commander   = &Fs{}
+	_ fs.Metadataer  = &Object{}
 )

backend/http/http_internal_test.go

@@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple {
 		what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)
 		assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])
 		assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])
+
+		// Set the content disposition header for the fifth file
+		// later we will check if it is set using the metadata method
+		if r.URL.Path == "/five.txt.gz" {
+			w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"")
+			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+			w.Header().Set("Cache-Control", "no-cache")
+			w.Header().Set("Content-Language", "en-US")
+			w.Header().Set("Content-Encoding", "gzip")
+		}
+
 		fileServer.ServeHTTP(w, r)
 	})
 
@@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 
 	sort.Sort(entries)
 
-	require.Equal(t, 4, len(entries))
+	require.Equal(t, 5, len(entries))
 
 	e := entries[0]
-	assert.Equal(t, "four", e.Remote())
+	assert.Equal(t, "five.txt.gz", e.Remote())
 	assert.Equal(t, int64(-1), e.Size())
-	_, ok := e.(fs.Directory)
+	_, ok := e.(fs.Object)
 	assert.True(t, ok)
 
 	e = entries[1]
+	assert.Equal(t, "four", e.Remote())
+	assert.Equal(t, int64(-1), e.Size())
+	_, ok = e.(fs.Directory)
+	assert.True(t, ok)
+
+	e = entries[2]
 	assert.Equal(t, "one%.txt", e.Remote())
 	assert.Equal(t, int64(5+lineEndSize), e.Size())
 	_, ok = e.(*Object)
 	assert.True(t, ok)
 
-	e = entries[2]
+	e = entries[3]
 	assert.Equal(t, "three", e.Remote())
 	assert.Equal(t, int64(-1), e.Size())
 	_, ok = e.(fs.Directory)
 	assert.True(t, ok)
 
-	e = entries[3]
+	e = entries[4]
 	assert.Equal(t, "two.html", e.Remote())
 	if noSlash {
 		assert.Equal(t, int64(-1), e.Size())
@@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) {
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
+func TestNewObjectWithMetadata(t *testing.T) {
+	f := prepare(t)
+	o, err := f.NewObject(context.Background(), "/five.txt.gz")
+	require.NoError(t, err)
+	assert.Equal(t, "five.txt.gz", o.Remote())
+	ho, ok := o.(*Object)
+	assert.True(t, ok)
+	metadata, err := ho.Metadata(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"])
+	assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"])
+	assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"])
+	assert.Equal(t, "no-cache", metadata["cache-control"])
+	assert.Equal(t, "en-US", metadata["content-language"])
+	assert.Equal(t, "gzip", metadata["content-encoding"])
+}
+
 func TestOpen(t *testing.T) {
 	m := prepareServer(t)
 

cmd/serve/serve.go

@@ -23,7 +23,15 @@ subcommand to specify the protocol, e.g.
 rclone serve http remote:
 ` + "```" + `
 
-Each subcommand has its own options which you can see in their help.`,
+When the "--metadata" flag is enabled, the following metadata fields will be provided as headers:
+- "content-disposition"
+- "cache-control" 
+- "content-language"
+- "content-encoding"
+Note: The availability of these fields depends on whether the remote supports metadata.
+
+Each subcommand has its own options which you can see in their help.
+`,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},

docs/content/overview.md

@@ -34,7 +34,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Photos                | -                 | -       | No               | Yes             | R         | -        |
 | HDFS                         | -                 | R/W     | No               | No              | -         | -        |
 | HiDrive                      | HiDrive ¹²        | R/W     | No               | No              | -         | -        |
-| HTTP                         | -                 | R       | No               | No              | R         | -        |
+| HTTP                         | -                 | R       | No               | No              | R         | R        |
 | iCloud Drive                 | -                 | R       | No               | No              | -         | -        |
 | Internet Archive             | MD5, SHA1, CRC32  | R/W ¹¹  | No               | No              | -         | RWU      |
 | Jottacloud                   | MD5               | R/W     | Yes              | No              | R         | RW       |

fs/object/object.go

@@ -191,11 +191,12 @@ var _ fs.Fs = MemoryFs
 
 // MemoryObject is an in memory object
 type MemoryObject struct {
-	remote  string
+	remote   string
-	modTime time.Time
+	modTime  time.Time
-	content []byte
+	content  []byte
-	meta    fs.Metadata
+	meta     fs.Metadata
-	fs      fs.Fs
+	fs       fs.Fs
+	mimeType string
 }
 
 // NewMemoryObject returns an in memory Object with the modTime and content passed in
@@ -214,6 +215,12 @@ func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject {
 	return o
 }
 
+// WithMimeType adds mimeType to the MemoryObject
+func (o *MemoryObject) WithMimeType(mimeType string) *MemoryObject {
+	o.mimeType = mimeType
+	return o
+}
+
 // Content returns the underlying buffer
 func (o *MemoryObject) Content() []byte {
 	return o.content
@@ -329,8 +336,14 @@ func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return o.meta, nil
 }
 
+// MimeType on the object
+func (o *MemoryObject) MimeType(ctx context.Context) string {
+	return o.mimeType
+}
+
 // Check interfaces
 var (
 	_ fs.Object     = (*MemoryObject)(nil)
+	_ fs.MimeTyper  = (*MemoryObject)(nil)
 	_ fs.Metadataer = (*MemoryObject)(nil)
 )

fs/object/object_test.go

@@ -87,6 +87,7 @@ func TestMemoryObject(t *testing.T) {
 	content = content[:6] // make some extra cap
 
 	o := object.NewMemoryObject(remote, now, content)
+	o.WithMimeType("text/plain; charset=utf-8")
 
 	assert.Equal(t, content, o.Content())
 	assert.Equal(t, object.MemoryFs, o.Fs())
@@ -95,6 +96,7 @@ func TestMemoryObject(t *testing.T) {
 	assert.Equal(t, now, o.ModTime(context.Background()))
 	assert.Equal(t, int64(len(content)), o.Size())
 	assert.Equal(t, true, o.Storable())
+	assert.Equal(t, "text/plain; charset=utf-8", o.MimeType(context.Background()))
 
 	Hash, err := o.Hash(context.Background(), hash.MD5)
 	assert.NoError(t, err)

lib/http/serve/serve.go

@@ -39,6 +39,26 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
 	modTime := o.ModTime(r.Context())
 	w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat))
 
+	// Set metadata headers if present
+	metadata, err := fs.GetMetadata(r.Context(), o)
+	if err != nil {
+		fs.Debugf(o, "Request get metadata error: %v", err)
+	}
+	if metadata != nil {
+		if metadata["content-disposition"] != "" {
+			w.Header().Set("Content-Disposition", metadata["content-disposition"])
+		}
+		if metadata["cache-control"] != "" {
+			w.Header().Set("Cache-Control", metadata["cache-control"])
+		}
+		if metadata["content-language"] != "" {
+			w.Header().Set("Content-Language", metadata["content-language"])
+		}
+		if metadata["content-encoding"] != "" {
+			w.Header().Set("Content-Encoding", metadata["content-encoding"])
+		}
+	}
+
 	if r.Method == "HEAD" {
 		return
 	}

lib/http/serve/serve_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest/mockobject"
 	"github.com/stretchr/testify/assert"
 )
@@ -82,3 +84,23 @@ func TestObjectBadRange(t *testing.T) {
 	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }
+
+func TestObjectHEADMetadata(t *testing.T) {
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("HEAD", "http://example.com/aFile", nil)
+	m := fs.Metadata{
+		"content-disposition": "inline",
+		"cache-control":       "no-cache",
+		"content-language":    "en",
+		"content-encoding":    "gzip",
+	}
+	o := object.NewMemoryObject("aFile", time.Now(), []byte("")).
+		WithMetadata(m).WithMimeType("text/plain; charset=utf-8")
+	Object(w, r, o)
+	resp := w.Result()
+	assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "inline", resp.Header.Get("Content-Disposition"))
+	assert.Equal(t, "no-cache", resp.Header.Get("Cache-Control"))
+	assert.Equal(t, "en", resp.Header.Get("Content-Language"))
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+}