From 8d353039a694fb277f505cb189424fe9a6b00320 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 11 Apr 2025 17:52:50 +0100 Subject: [PATCH] log: add log rotation to --log-file - fixes #2259 --- docs/content/docs.md | 51 +++++++++++++++++++++++++++ fs/log/log.go | 82 ++++++++++++++++++++++++++++++++++++-------- go.mod | 1 + go.sum | 2 ++ 4 files changed, 121 insertions(+), 15 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index de5b9590d..ba9435781 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1727,6 +1727,57 @@ Note that if you are using the `logrotate` program to manage rclone's logs, then you should use the `copytruncate` option as rclone doesn't have a signal to rotate logs. +Alternatively you can use the options below to manage rclone's built +in log rotation. + +### --log-file-max-size SizeSuffix + +Maximum size of the log file before it's rotated (eg `10M`). This SizeSuffix +is rounded to the nearest MiB or 1 MiB if lower. + +If `--log-file` is not set then this option will be ignored. + +If this option is not set, then the other log rotation options will be +ignored. + +For example if the following flags are in use + +```sh +rclone --log-file rclone.log --log-file-max-size 1M --log-file-max-backups 3 +``` + +Then this will create log files which look like this + +```console +$ ls -l +-rw------- 1 user user 1048491 Apr 11 17:15 rclone-2025-04-11T17-15-29.998.log +-rw------- 1 user user 1048511 Apr 11 17:15 rclone-2025-04-11T17-15-30.467.log +-rw------- 1 user user 1048559 Apr 11 17:15 rclone-2025-04-11T17-15-30.543.log +-rw------- 1 user user 521602 Apr 11 17:15 rclone.log +``` + +The file `rclone.log` being the current one. + +### --log-file-compress + +If set, compress rotated log files using gzip. This changes the +extension of the old log files to `.log.gz`. + +Defaults to false - don't compress log files. + +### --log-file-max-age Duration + +Maximum duration to retain old log files (eg `7d`). This is rounded to +the dearest day, or 1 day if lower. + +The default is to retain all old log files. + +### --log-file-max-backups int + +Maximum number of old log files to retain + +The default is to retain all old log files. + ### --log-format string Comma separated list of log format options. The accepted options are: diff --git a/fs/log/log.go b/fs/log/log.go index 8e56a2d18..0c0fad727 100644 --- a/fs/log/log.go +++ b/fs/log/log.go @@ -9,8 +9,10 @@ import ( "reflect" "runtime" "strings" + "time" "github.com/rclone/rclone/fs" + "gopkg.in/natefinch/lumberjack.v2" ) // OptionsInfo descripts the Options in use @@ -19,6 +21,26 @@ var OptionsInfo = fs.Options{{ Default: "", Help: "Log everything to this file", Groups: "Logging", +}, { + Name: "log_file_max_size", + Default: fs.SizeSuffix(-1), + Help: `Maximum size of the log file before it's rotated (eg "10M")`, + Groups: "Logging", +}, { + Name: "log_file_max_backups", + Default: 0, + Help: "Maximum number of old log files to retain.", + Groups: "Logging", +}, { + Name: "log_file_max_age", + Default: fs.Duration(0), + Help: `Maximum duration to retain old log files (eg "7d")`, + Groups: "Logging", +}, { + Name: "log_file_compress", + Default: false, + Help: "If set, compress rotated log files using gzip.", + Groups: "Logging", }, { Name: "log_format", Default: logFormatDate | logFormatTime, @@ -54,12 +76,16 @@ var OptionsInfo = fs.Options{{ // Options contains options for controlling the logging type Options struct { - File string `config:"log_file"` // Log everything to this file - Format logFormat `config:"log_format"` // Comma separated list of log format options - UseSyslog bool `config:"syslog"` // Use Syslog for logging - SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,... - LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging - WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"` + File string `config:"log_file"` // Log everything to this file + MaxSize fs.SizeSuffix `config:"log_file_max_size"` // Max size of log file + MaxBackups int `config:"log_file_max_backups"` // Max backups of log file + MaxAge fs.Duration `config:"log_file_max_age"` // Max age of of log file + Compress bool `config:"log_file_compress"` // Set to compress log file + Format logFormat `config:"log_format"` // Comma separated list of log format options + UseSyslog bool `config:"syslog"` // Use Syslog for logging + SyslogFacility string `config:"syslog_facility"` // Facility for syslog, e.g. KERN,USER,... + LogSystemdSupport bool `config:"log_systemd"` // set if using systemd logging + WindowsEventLogLevel fs.LogLevel `config:"windows_event_log_level"` } func init() { @@ -182,16 +208,42 @@ func InitLogging() { // Log file output if Opt.File != "" { - f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640) - if err != nil { - fs.Fatalf(nil, "Failed to open log file: %v", err) + var w io.Writer + if Opt.MaxSize == 0 { + // No log rotation - just open the file as normal + // We'll capture tracebacks like this too. + f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640) + if err != nil { + fs.Fatalf(nil, "Failed to open log file: %v", err) + } + _, err = f.Seek(0, io.SeekEnd) + if err != nil { + fs.Errorf(nil, "Failed to seek log file to end: %v", err) + } + redirectStderr(f) + w = f + } else { + // Round with a minimum of 1 if set + round := func(x float64) int { + if x <= 0 { + return 0 + } else if x <= 1 { + return 1 + } + return int(x + 0.5) + } + // Log rotation active + f := &lumberjack.Logger{ + Filename: Opt.File, + MaxSize: round(float64(Opt.MaxSize) / float64(fs.Mebi)), // MiB + MaxBackups: Opt.MaxBackups, + MaxAge: round(time.Duration(Opt.MaxAge).Hours() / 24), // Days + Compress: Opt.Compress, + LocalTime: true, // format log file names in localtime + } + w = f } - _, err = f.Seek(0, io.SeekEnd) - if err != nil { - fs.Errorf(nil, "Failed to seek log file to end: %v", err) - } - redirectStderr(f) - Handler.setWriter(f) + Handler.setWriter(w) } // --use-json-log implies JSON formatting diff --git a/go.mod b/go.mod index 72e0bb01f..106a293a8 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( golang.org/x/text v0.28.0 golang.org/x/time v0.12.0 google.golang.org/api v0.247.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/validator.v2 v2.0.1 gopkg.in/yaml.v3 v3.0.1 storj.io/uplink v1.13.1 diff --git a/go.sum b/go.sum index c7c752ec1..f4ce2432b 100644 --- a/go.sum +++ b/go.sum @@ -1045,6 +1045,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY=