From 139b84393089f708d32dc136432596f7e1802ac5 Mon Sep 17 00:00:00 2001 From: lezhnev Date: Sat, 26 Oct 2024 16:20:31 +0500 Subject: [PATCH] enable logging control --- common/util.go | 14 ++++++++------ ingest/discover.go | 3 ++- ingest/ingest.go | 5 ++--- test_util/test_utils.go | 3 ++- ui/console.go | 14 ++++++++++---- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/common/util.go b/common/util.go index e10dcaa..703bd65 100644 --- a/common/util.go +++ b/common/util.go @@ -107,13 +107,13 @@ func PrintMem(db *sql.DB) (rss uint64) { if totalBlocks > 0 { freeBlocksPct = int(float64(freeBlocks) / float64(totalBlocks) * 100) } - log.Printf("DuckDB: fileSize:%s, walSize:%s, mem:%s/%s, Blcs[total/used/free/freePcs]:%d,%d,%d,%d%% ", + Out("DuckDB: fileSize:%s, walSize:%s, mem:%s/%s, Blcs[total/used/free/freePcs]:%d,%d,%d,%d%% ", dbSize, walSize, memSize, memLimit, totalBlocks, usedBlocks, freeBlocks, freeBlocksPct) var m runtime.MemStats runtime.ReadMemStats(&m) - log.Printf( + Out( "System: %s, %s", fmt.Sprintf("RSS:%dMiB", m.Sys/1024/1024), // total virtual memory reserved from OS fmt.Sprintf("HeapAlloc:%dMiB", m.HeapAlloc/1024/1024), // HeapAlloc is bytes of allocated heap objects. @@ -124,8 +124,10 @@ func PrintMem(db *sql.DB) (rss uint64) { var EnableLogging bool -func Log(message string) { - fmt.Println(message) +func Out(pattern string, args ...any) { + if EnableLogging { + log.Printf(pattern, args...) + } } //func CleanMem() { @@ -136,9 +138,9 @@ func Log(message string) { // ctx := context.Background() // cmd := exec.CommandContext(ctx, "bash", "-c", `echo 3 > /proc/sys/vm/drop_caches`) // out, err := cmd.CombinedOutput() -// log.Printf("cleanmem: %s", out) +// Out("cleanmem: %s", out) // if err != nil { -// log.Printf("cleanmem error: %s", err) +// Out("cleanmem error: %s", err) // return // } //} diff --git a/ingest/discover.go b/ingest/discover.go index 49b854c..af31cc3 100644 --- a/ingest/discover.go +++ b/ingest/discover.go @@ -2,6 +2,7 @@ package ingest import ( "golang.org/x/xerrors" + "heaplog_2024/common" "heaplog_2024/db" "log" "path/filepath" @@ -48,7 +49,7 @@ func (d *Discover) DiscoverFiles() (news, obsoletes []string, err error) { for i, _ := range obsoletes { report += "- " + obsoletes[i] + "\n" } - log.Printf(report) + common.Out(report) } return diff --git a/ingest/ingest.go b/ingest/ingest.go index ba2d450..b2d48ce 100644 --- a/ingest/ingest.go +++ b/ingest/ingest.go @@ -11,7 +11,6 @@ import ( "heaplog_2024/db" "heaplog_2024/scanner" "io" - "log" "os" "slices" "time" @@ -223,7 +222,7 @@ func (ing *Ingest) saveBatch(file string, messages <-chan *ScannedTokenizedMessa // Report if curSegmentMessages > 0 { - log.Printf("indexed %s[%d:%d]: %d messages, %d terms in %s", file, curSegment.Loc.From, curSegment.Loc.To, curSegmentMessages, len(segmentTerms), time.Now().Sub(t0).String()) + common.Out("indexed %s[%d:%d]: %d messages, %d terms in %s", file, curSegment.Loc.From, curSegment.Loc.To, curSegmentMessages, len(segmentTerms), time.Now().Sub(t0).String()) } // Cleanup @@ -330,7 +329,7 @@ func (ing *Ingest) readMessagesInStream(name string, stream io.ReaderAt, message messageLen := layout.To - layout.From if messageLen > maxIndexableSize { - log.Printf("big message %dMiB at %s:%d", messageLen/1024/1024, name, layout.From) + common.Out("big message %dMiB at %s:%d", messageLen/1024/1024, name, layout.From) // Index only the beginning and the end of the big message. halfSize := maxIndexableSize / 2 diff --git a/test_util/test_utils.go b/test_util/test_utils.go index d9a90a6..e52bb21 100644 --- a/test_util/test_utils.go +++ b/test_util/test_utils.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "github.com/prometheus/procfs" + "heaplog_2024/common" "log" "os" "path" @@ -107,5 +108,5 @@ func ProcStat() { if err != nil { log.Fatalf("could not get process stat: %s", err) } - log.Printf("RSS: %dMb\n", stat.ResidentMemory()/1024/1024) + common.Out("RSS: %dMb\n", stat.ResidentMemory()/1024/1024) } diff --git a/ui/console.go b/ui/console.go index a246ddc..c2c848f 100644 --- a/ui/console.go +++ b/ui/console.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" + "heaplog_2024/common" "log" "os" "os/signal" @@ -38,6 +39,7 @@ func overrideConfig(cfg Config, ctx *cli.Context) Config { if ctx.Int("DuckdbMaxMemMb") != 0 { cfg.DuckdbMaxMemMb = uint(ctx.Int("DuckdbMaxMemMb")) } + cfg.EnableLogging = ctx.Bool("Verbose") return cfg } @@ -47,6 +49,9 @@ func PrepareConsoleApp() (app *cli.App) { prepareCfg := func(ctx *cli.Context) (Config, error) { cfg, _ := LoadConfig(true) cfg = overrideConfig(cfg, ctx) + + common.EnableLogging = cfg.EnableLogging + return cfg, cfg.Validate() } @@ -92,8 +97,9 @@ func PrepareConsoleApp() (app *cli.App) { Usage: "Max memory the duckdb instance is allowed to allocate (Mb)", }, &cli.BoolFlag{ + Name: "Verbose", Aliases: []string{"v"}, - Usage: "Show extra details about what the service does", + Usage: "Show extra details about what the service is doing", }, } @@ -122,18 +128,18 @@ func PrepareConsoleApp() (app *cli.App) { httpApp := makeHttpApp(happ, "") sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) go func() { <-sigs cancel() // stop the program log.Printf("Stopping heaplog...") - t := time.Second * 10 // the same as the docker's timeout for "docker stop" + t := time.Second * 3 time.Sleep(t) err := httpApp.Shutdown() if err != nil { - log.Printf("%s", err) + common.Out("%s", err) } }()