From f360532eb12c92546ec88672508482c5389dab19 Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Mon, 26 Aug 2024 18:43:42 -0400 Subject: [PATCH 1/4] lnd+build: add logcompressor flag --- build/log.go | 26 ++++++++++++++++++++++++++ config.go | 8 ++++++++ sample-lnd.conf | 3 +++ 3 files changed, 37 insertions(+) diff --git a/build/log.go b/build/log.go index 21c8b953e4..484b48e422 100644 --- a/build/log.go +++ b/build/log.go @@ -36,6 +36,32 @@ func (t LogType) String() string { } } +// Declare the supported log file compressors as exported consts for easier use +// from other projects. +const ( + // Gzip is the default compressor. + Gzip = "gzip" + + // Zstd is a modern compressor that compresses better than Gzip, in less + // time. + Zstd = "zstd" +) + +// logCompressors maps the identifier for each supported compression algorithm +// to the extension used for the compressed log files. +var logCompressors = map[string]string{ + Gzip: "gz", + Zstd: "zst", +} + +// SuportedLogCompressor returns whether or not logCompressor is a supported +// compression algorithm for log files. +func SuportedLogCompressor(logCompressor string) bool { + _, ok := logCompressors[logCompressor] + + return ok +} + // LogWriter is a stub type whose behavior can be changed using the build flags // "stdlog" and "nolog". The default behavior is to write to both stdout and the // RotatorPipe. Passing "stdlog" will cause it only to write to stdout, and diff --git a/config.go b/config.go index 34d84720cb..dada30ee00 100644 --- a/config.go +++ b/config.go @@ -59,6 +59,7 @@ const ( defaultLogLevel = "info" defaultLogDirname = "logs" defaultLogFilename = "lnd.log" + defaultLogCompressor = build.Gzip defaultRPCPort = 10009 defaultRESTPort = 8080 defaultPeerPort = 9735 @@ -315,6 +316,7 @@ type Config struct { ReadMacPath string `long:"readonlymacaroonpath" description:"Path to write the read-only macaroon for lnd's RPC and REST services if it doesn't exist"` InvoiceMacPath string `long:"invoicemacaroonpath" description:"Path to the invoice-only macaroon for lnd's RPC and REST services if it doesn't exist"` LogDir string `long:"logdir" description:"Directory to log output."` + LogCompressor string `long:"logcompressor" description:"Compression algorithm to use when rotating logs." choice:"gzip" choice:"zstd"` MaxLogFiles int `long:"maxlogfiles" description:"Maximum logfiles to keep (0 for no rotation)"` MaxLogFileSize int `long:"maxlogfilesize" description:"Maximum logfile size in MB"` AcceptorTimeout time.Duration `long:"acceptortimeout" description:"Time after which an RPCAcceptor will time out and return false if it hasn't yet received a response"` @@ -560,6 +562,7 @@ func DefaultConfig() Config { LetsEncryptDir: defaultLetsEncryptDir, LetsEncryptListen: defaultLetsEncryptListen, LogDir: defaultLogDir, + LogCompressor: defaultLogCompressor, MaxLogFiles: defaultMaxLogFiles, MaxLogFileSize: defaultMaxLogFileSize, AcceptorTimeout: defaultAcceptorTimeout, @@ -1446,6 +1449,11 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser, os.Exit(0) } + if !build.SuportedLogCompressor(cfg.LogCompressor) { + return nil, mkErr("invalid log compressor: %v", + cfg.LogCompressor) + } + // Initialize logging at the default logging level. SetupLoggers(cfg.LogWriter, interceptor) err = cfg.LogWriter.InitLogRotator( diff --git a/sample-lnd.conf b/sample-lnd.conf index c9fb0e4f8c..56d66b5951 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -36,6 +36,9 @@ ; Max log file size in MB before it is rotated. ; maxlogfilesize=10 +; Compression algorithm to use when rotating logs. +; logcompressor=gzip + ; Time after which an RPCAcceptor will time out and return false if ; it hasn't yet received a response. ; acceptortimeout=15s From 2f71822fe0d07c16bcacf8224be0a57267307c75 Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Mon, 26 Aug 2024 18:46:07 -0400 Subject: [PATCH 2/4] mod: bump logrotate to v1.1.2 --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3f22cdedfb..649faafe7f 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad github.com/jedib0t/go-pretty/v6 v6.2.7 github.com/jessevdk/go-flags v1.4.0 - github.com/jrick/logrotate v1.0.0 + github.com/jrick/logrotate v1.1.2 github.com/kkdai/bstream v1.0.0 github.com/lightninglabs/neutrino v0.16.1-0.20240425105051-602843d34ffd github.com/lightninglabs/neutrino/cache v1.1.2 diff --git a/go.sum b/go.sum index f937d69e0e..2206c00e13 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= +github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= From b3c25166edd51ad05ae60c9f4c67d146114cabd3 Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Mon, 26 Aug 2024 18:59:52 -0400 Subject: [PATCH 3/4] build+mod: add zstd compressor support --- build/logrotator.go | 27 +++++++++++++++++++++++++-- config.go | 2 +- go.mod | 1 + go.sum | 2 ++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/build/logrotator.go b/build/logrotator.go index b6e697f9ba..aa4683bba2 100644 --- a/build/logrotator.go +++ b/build/logrotator.go @@ -1,6 +1,7 @@ package build import ( + "compress/gzip" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "github.com/btcsuite/btclog" "github.com/jrick/logrotate/rotator" + "github.com/klauspost/compress/zstd" ) // RotatingLogWriter is a wrapper around the LogWriter that supports log file @@ -58,8 +60,8 @@ func (r *RotatingLogWriter) RegisterSubLogger(subsystem string, // InitLogRotator initializes the log file rotator to write logs to logFile and // create roll files in the same directory. It should be called as early on // startup and possible and must be closed on shutdown by calling `Close`. -func (r *RotatingLogWriter) InitLogRotator(logFile string, maxLogFileSize int, - maxLogFiles int) error { +func (r *RotatingLogWriter) InitLogRotator(logFile, logCompressor string, + maxLogFileSize int, maxLogFiles int) error { logDir, _ := filepath.Split(logFile) err := os.MkdirAll(logDir, 0700) @@ -73,6 +75,27 @@ func (r *RotatingLogWriter) InitLogRotator(logFile string, maxLogFileSize int, return fmt.Errorf("failed to create file rotator: %w", err) } + // Reject unknown compressors. + if !SuportedLogCompressor(logCompressor) { + return fmt.Errorf("unknown log compressor: %v", logCompressor) + } + + var c rotator.Compressor + switch logCompressor { + case Gzip: + c = gzip.NewWriter(nil) + + case Zstd: + c, err = zstd.NewWriter(nil) + if err != nil { + return fmt.Errorf("failed to create zstd compressor: "+ + "%w", err) + } + } + + // Apply the compressor and its file suffix to the log rotator. + r.logRotator.SetCompressor(c, logCompressors[logCompressor]) + // Run rotator as a goroutine now but make sure we catch any errors // that happen in case something with the rotation goes wrong during // runtime (like running out of disk space or not being allowed to diff --git a/config.go b/config.go index dada30ee00..513c217e69 100644 --- a/config.go +++ b/config.go @@ -1458,7 +1458,7 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser, SetupLoggers(cfg.LogWriter, interceptor) err = cfg.LogWriter.InitLogRotator( filepath.Join(cfg.LogDir, defaultLogFilename), - cfg.MaxLogFileSize, cfg.MaxLogFiles, + cfg.LogCompressor, cfg.MaxLogFileSize, cfg.MaxLogFiles, ) if err != nil { str := "log rotation setup failed: %v" diff --git a/go.mod b/go.mod index 649faafe7f..65a4f991fc 100644 --- a/go.mod +++ b/go.mod @@ -118,6 +118,7 @@ require ( github.com/json-iterator/go v1.1.11 // indirect github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 // indirect github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect + github.com/klauspost/compress v1.17.9 github.com/lib/pq v1.10.9 // indirect github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 2206c00e13..59e0c9faf4 100644 --- a/go.sum +++ b/go.sum @@ -419,6 +419,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kkdai/bstream v1.0.0 h1:Se5gHwgp2VT2uHfDrkbbgbgEvV9cimLELwrPJctSjg8= github.com/kkdai/bstream v1.0.0/go.mod h1:FDnDOHt5Yx4p3FaHcioFT0QjDOtgUpvjeZqAs+NVZZA= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 0a451b6d361db50cbc12fcefea54ee9046750e02 Mon Sep 17 00:00:00 2001 From: Jonathan Harvey-Buschel Date: Thu, 12 Sep 2024 12:29:17 -0400 Subject: [PATCH 4/4] docs: add release notes --- docs/release-notes/release-notes-0.19.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/release-notes-0.19.0.md b/docs/release-notes/release-notes-0.19.0.md index 919d25aa49..bda488095a 100644 --- a/docs/release-notes/release-notes-0.19.0.md +++ b/docs/release-notes/release-notes-0.19.0.md @@ -36,6 +36,8 @@ # Improvements ## Functional Updates +* [Allow](https://github.com/lightningnetwork/lnd/pull/9017) the compression of logs during rotation with ZSTD via the `logcompressor` startup argument. + ## RPC Updates ## lncli Updates @@ -45,6 +47,8 @@ ## Breaking Changes ## Performance Improvements +* Log rotation can now use ZSTD + # Technical and Architectural Updates ## BOLT Spec Updates