log: add log rotation to --log-file - fixes #2259

docs/content/docs.md

@@ -1727,6 +1727,57 @@ Note that if you are using the `logrotate` program to manage rclone's
 logs, then you should use the `copytruncate` option as rclone doesn't
 have a signal to rotate logs.
 
+Alternatively you can use the options below to manage rclone's built
+in log rotation.
+
+### --log-file-max-size SizeSuffix
+
+Maximum size of the log file before it's rotated (eg `10M`). This SizeSuffix
+is rounded to the nearest MiB or 1 MiB if lower.
+
+If `--log-file` is not set then this option will be ignored.
+
+If this option is not set, then the other log rotation options will be
+ignored.
+
+For example if the following flags are in use
+
+```sh
+rclone --log-file rclone.log --log-file-max-size 1M --log-file-max-backups 3
+```
+
+Then this will create log files which look like this
+
+```console
+$ ls -l
+-rw-------  1 user user  1048491 Apr 11 17:15 rclone-2025-04-11T17-15-29.998.log
+-rw-------  1 user user  1048511 Apr 11 17:15 rclone-2025-04-11T17-15-30.467.log
+-rw-------  1 user user  1048559 Apr 11 17:15 rclone-2025-04-11T17-15-30.543.log
+-rw-------  1 user user   521602 Apr 11 17:15 rclone.log
+```
+
+The file `rclone.log` being the current one.
+
+### --log-file-compress
+
+If set, compress rotated log files using gzip. This changes the
+extension of the old log files to `.log.gz`.
+
+Defaults to false - don't compress log files.
+
+### --log-file-max-age Duration
+
+Maximum duration to retain old log files (eg `7d`). This is rounded to
+the dearest day, or 1 day if lower.
+
+The default is to retain all old log files.
+
+### --log-file-max-backups int
+
+Maximum number of old log files to retain
+
+The default is to retain all old log files.
+
 ### --log-format string
 
 Comma separated list of log format options. The accepted options are:

fs/log/log.go

@@ -9,8 +9,10 @@ import (
 	"reflect"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/rclone/rclone/fs"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 // OptionsInfo descripts the Options in use
@@ -19,6 +21,26 @@ var OptionsInfo = fs.Options{{
 	Default: "",
 	Help:    "Log everything to this file",
 	Groups:  "Logging",
+}, {
+	Name:    "log_file_max_size",
+	Default: fs.SizeSuffix(-1),
+	Help:    `Maximum size of the log file before it's rotated (eg "10M")`,
+	Groups:  "Logging",
+}, {
+	Name:    "log_file_max_backups",
+	Default: 0,
+	Help:    "Maximum number of old log files to retain.",
+	Groups:  "Logging",
+}, {
+	Name:    "log_file_max_age",
+	Default: fs.Duration(0),
+	Help:    `Maximum duration to retain old log files (eg "7d")`,
+	Groups:  "Logging",
+}, {
+	Name:    "log_file_compress",
+	Default: false,
+	Help:    "If set, compress rotated log files using gzip.",
+	Groups:  "Logging",
 }, {
 	Name:    "log_format",
 	Default: logFormatDate | logFormatTime,
@@ -54,12 +76,16 @@ var OptionsInfo = fs.Options{{
 
 // Options contains options for controlling the logging
 type Options struct {
-	File                 string      `config:"log_file"`        // Log everything to this file
+	File                 string        `config:"log_file"`             // Log everything to this file
-	Format               logFormat   `config:"log_format"`      // Comma separated list of log format options
+	MaxSize              fs.SizeSuffix `config:"log_file_max_size"`    // Max size of log file
-	UseSyslog            bool        `config:"syslog"`          // Use Syslog for logging
+	MaxBackups           int           `config:"log_file_max_backups"` // Max backups of log file
-	SyslogFacility       string      `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,...
+	MaxAge               fs.Duration   `config:"log_file_max_age"`     // Max age of of log file
-	LogSystemdSupport    bool        `config:"log_systemd"`     // set if using systemd logging
+	Compress             bool          `config:"log_file_compress"`    // Set to compress log file
-	WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"`
+	Format               logFormat     `config:"log_format"`           // Comma separated list of log format options
+	UseSyslog            bool          `config:"syslog"`               // Use Syslog for logging
+	SyslogFacility       string        `config:"syslog_facility"`      // Facility for syslog, e.g. KERN,USER,...
+	LogSystemdSupport    bool          `config:"log_systemd"`          // set if using systemd logging
+	WindowsEventLogLevel fs.LogLevel   `config:"windows_event_log_level"`
 }
 
 func init() {
@@ -182,16 +208,42 @@ func InitLogging() {
 
 	// Log file output
 	if Opt.File != "" {
-		f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
+		var w io.Writer
-		if err != nil {
+		if Opt.MaxSize == 0 {
-			fs.Fatalf(nil, "Failed to open log file: %v", err)
+			// No log rotation - just open the file as normal
+			// We'll capture tracebacks like this too.
+			f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
+			if err != nil {
+				fs.Fatalf(nil, "Failed to open log file: %v", err)
+			}
+			_, err = f.Seek(0, io.SeekEnd)
+			if err != nil {
+				fs.Errorf(nil, "Failed to seek log file to end: %v", err)
+			}
+			redirectStderr(f)
+			w = f
+		} else {
+			// Round with a minimum of 1 if set
+			round := func(x float64) int {
+				if x <= 0 {
+					return 0
+				} else if x <= 1 {
+					return 1
+				}
+				return int(x + 0.5)
+			}
+			// Log rotation active
+			f := &lumberjack.Logger{
+				Filename:   Opt.File,
+				MaxSize:    round(float64(Opt.MaxSize) / float64(fs.Mebi)), // MiB
+				MaxBackups: Opt.MaxBackups,
+				MaxAge:     round(time.Duration(Opt.MaxAge).Hours() / 24), // Days
+				Compress:   Opt.Compress,
+				LocalTime:  true, // format log file names in localtime
+			}
+			w = f
 		}
-		_, err = f.Seek(0, io.SeekEnd)
+		Handler.setWriter(w)
-		if err != nil {
-			fs.Errorf(nil, "Failed to seek log file to end: %v", err)
-		}
-		redirectStderr(f)
-		Handler.setWriter(f)
 	}
 
 	// --use-json-log implies JSON formatting

go.mod

@@ -89,6 +89,7 @@ require (
 	golang.org/x/text v0.28.0
 	golang.org/x/time v0.12.0
 	google.golang.org/api v0.247.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/validator.v2 v2.0.1
 	gopkg.in/yaml.v3 v3.0.1
 	storj.io/uplink v1.13.1

go.sum

@@ -1045,6 +1045,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=