diff --git a/cmd/bfgd/bfgd.go b/cmd/bfgd/bfgd.go index 8ddede686..66fb668d8 100644 --- a/cmd/bfgd/bfgd.go +++ b/cmd/bfgd/bfgd.go @@ -81,6 +81,12 @@ var ( Help: "address and port bfgd prometheus listens on", Print: config.PrintAll, }, + "BFG_PPROF_ADDRESS": config.Config{ + Value: &cfg.PprofListenAddress, + DefaultValue: "", + Help: "address and port bfgd pprof listens on (open
/debug/pprof to see available profiles)", + Print: config.PrintAll, + }, } ) diff --git a/cmd/bssd/bssd.go b/cmd/bssd/bssd.go index 33ffdcc24..4d214a081 100644 --- a/cmd/bssd/bssd.go +++ b/cmd/bssd/bssd.go @@ -56,6 +56,12 @@ var ( Help: "address and port bssd prometheus listens on", Print: config.PrintAll, }, + "BSS_PPROF_ADDRESS": config.Config{ + Value: &cfg.PprofListenAddress, + DefaultValue: "", + Help: "address and port bssd pprof listens on (open /debug/pprof to see available profiles)", + Print: config.PrintAll, + }, } ) diff --git a/cmd/popmd/popmd.go b/cmd/popmd/popmd.go index cb130cca7..ab8ca8156 100644 --- a/cmd/popmd/popmd.go +++ b/cmd/popmd/popmd.go @@ -58,7 +58,13 @@ var ( "POPM_PROMETHEUS_ADDRESS": config.Config{ Value: &cfg.PrometheusListenAddress, DefaultValue: "", - Help: "address and port bssd prometheus listens on", + Help: "address and port popm prometheus listens on", + Print: config.PrintAll, + }, + "POPM_PPROF_ADDRESS": config.Config{ + Value: &cfg.PrometheusListenAddress, + DefaultValue: "", + Help: "address and port popm pprof listens on (open /debug/pprof to see available profiles)", Print: config.PrintAll, }, "POPM_REMINE_THRESHOLD": config.Config{ diff --git a/cmd/tbcd/tbcd.go b/cmd/tbcd/tbcd.go index 997a14e56..1c6a07ac7 100644 --- a/cmd/tbcd/tbcd.go +++ b/cmd/tbcd/tbcd.go @@ -81,6 +81,12 @@ var ( Help: "address and port tbcd prometheus listens on", Print: config.PrintAll, }, + "TBC_PPROF_ADDRESS": config.Config{ + Value: &cfg.PprofListenAddress, + DefaultValue: "", + Help: "address and port tbcd pprof listens on (open /debug/pprof to see available profiles)", + Print: config.PrintAll, + }, } ) diff --git a/service/bfg/bfg.go b/service/bfg/bfg.go index 1d082448c..723145373 100644 --- a/service/bfg/bfg.go +++ b/service/bfg/bfg.go @@ -37,6 +37,7 @@ import ( "github.com/hemilabs/heminetwork/hemi/electrumx" "github.com/hemilabs/heminetwork/hemi/pop" "github.com/hemilabs/heminetwork/service/deucalion" + "github.com/hemilabs/heminetwork/service/pprof" ) // XXX this code needs to be a bit smarter when syncing bitcoin. We should @@ -104,6 +105,7 @@ type Config struct { LogLevel string PgURI string PrometheusListenAddress string + PprofListenAddress string PublicKeyAuth bool } @@ -1443,13 +1445,32 @@ func (s *Server) Run(pctx context.Context) error { log.Infof("public RPC Server shutdown cleanly") }() + // pprof + if s.cfg.PprofListenAddress != "" { + p, err := pprof.NewServer(&pprof.Config{ + ListenAddress: s.cfg.PprofListenAddress, + }) + if err != nil { + return fmt.Errorf("create pprof server: %w", err) + } + s.wg.Add(1) + go func() { + defer s.wg.Done() + if err := p.Run(ctx); !errors.Is(err, context.Canceled) { + log.Errorf("pprof server terminated with error: %v", err) + return + } + log.Infof("pprof server clean shutdown") + }() + } + // Prometheus if s.cfg.PrometheusListenAddress != "" { d, err := deucalion.New(&deucalion.Config{ ListenAddress: s.cfg.PrometheusListenAddress, }) if err != nil { - return fmt.Errorf("create server: %w", err) + return fmt.Errorf("create prometheus server: %w", err) } cs := []prometheus.Collector{ s.cmdsProcessed, // XXX should we make two counters? priv/pub diff --git a/service/bss/bss.go b/service/bss/bss.go index 66df3673f..c7bf2e48b 100644 --- a/service/bss/bss.go +++ b/service/bss/bss.go @@ -27,6 +27,7 @@ import ( "github.com/hemilabs/heminetwork/ethereum" "github.com/hemilabs/heminetwork/hemi" "github.com/hemilabs/heminetwork/service/deucalion" + "github.com/hemilabs/heminetwork/service/pprof" ) const ( @@ -61,6 +62,7 @@ type Config struct { ListenAddress string LogLevel string PrometheusListenAddress string + PprofListenAddress string } type Server struct { @@ -746,6 +748,25 @@ func (s *Server) Run(parrentCtx context.Context) error { s.wg.Add(1) go s.bfg(ctx) // Attempt to talk to bfg + // pprof + if s.cfg.PprofListenAddress != "" { + p, err := pprof.NewServer(&pprof.Config{ + ListenAddress: s.cfg.PprofListenAddress, + }) + if err != nil { + return fmt.Errorf("create pprof server: %w", err) + } + s.wg.Add(1) + go func() { + defer s.wg.Done() + if err := p.Run(ctx); !errors.Is(err, context.Canceled) { + log.Errorf("pprof server terminated with error: %v", err) + return + } + log.Infof("pprof server clean shutdown") + }() + } + // Prometheus if s.cfg.PrometheusListenAddress != "" { d, err := deucalion.New(&deucalion.Config{ diff --git a/service/popm/popm.go b/service/popm/popm.go index 44292f415..f9481dc8c 100644 --- a/service/popm/popm.go +++ b/service/popm/popm.go @@ -33,6 +33,7 @@ import ( "github.com/hemilabs/heminetwork/bitcoin" "github.com/hemilabs/heminetwork/hemi" "github.com/hemilabs/heminetwork/hemi/pop" + "github.com/hemilabs/heminetwork/service/pprof" ) // XXX we should debate if we can make pop miner fully transient. It feels like @@ -72,6 +73,8 @@ type Config struct { PrometheusListenAddress string + PprofListenAddress string + RetryMineThreshold uint StaticFee uint @@ -847,6 +850,25 @@ func (m *Miner) Run(pctx context.Context) error { } } + // pprof + if m.cfg.PprofListenAddress != "" { + p, err := pprof.NewServer(&pprof.Config{ + ListenAddress: m.cfg.PprofListenAddress, + }) + if err != nil { + return fmt.Errorf("create pprof server: %w", err) + } + m.wg.Add(1) + go func() { + defer m.wg.Done() + if err := p.Run(ctx); !errors.Is(err, context.Canceled) { + log.Errorf("pprof server terminated with error: %v", err) + return + } + log.Infof("pprof server clean shutdown") + }() + } + log.Infof("Starting PoP miner with BTC address %v (public key %x)", m.btcAddress.EncodeAddress(), m.btcPublicKey.SerializeCompressed()) diff --git a/service/pprof/pprof.go b/service/pprof/pprof.go new file mode 100644 index 000000000..393d90289 --- /dev/null +++ b/service/pprof/pprof.go @@ -0,0 +1,73 @@ +// Copyright (c) 2024 Hemi Labs, Inc. +// Use of this source code is governed by the MIT License, +// which can be found in the LICENSE file. + +package pprof + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/pprof" + "sync/atomic" + + "github.com/juju/loggo" +) + +var log = loggo.GetLogger("pprof") + +type Config struct { + ListenAddress string +} + +type Server struct { + cfg *Config + running atomic.Bool +} + +func NewServer(cfg *Config) (*Server, error) { + return &Server{cfg: cfg}, nil +} + +func (s *Server) Run(ctx context.Context) error { + if !s.running.CompareAndSwap(false, true) { + return errors.New("already running") + } + defer s.running.CompareAndSwap(true, false) + + if s.cfg.ListenAddress == "" { + return errors.New("listen address is required") + } + + pprofMux := http.NewServeMux() + pprofMux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + pprofMux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + pprofMux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + pprofMux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + pprofMux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + + pprofHttpServer := &http.Server{ + Addr: s.cfg.ListenAddress, + Handler: pprofMux, + BaseContext: func(net.Listener) context.Context { return ctx }, + } + + httpErrCh := make(chan error) + go func() { + log.Infof("pprof listening: %s", s.cfg.ListenAddress) + httpErrCh <- pprofHttpServer.ListenAndServe() + }() + defer func() { + if err := pprofHttpServer.Shutdown(ctx); err != nil { + log.Errorf("pprof http server exit: %v", err) + } + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-httpErrCh: + return err + } +} diff --git a/service/tbc/tbc.go b/service/tbc/tbc.go index 92049c85f..f8dac1812 100644 --- a/service/tbc/tbc.go +++ b/service/tbc/tbc.go @@ -39,6 +39,7 @@ import ( "github.com/hemilabs/heminetwork/database/tbcd" "github.com/hemilabs/heminetwork/database/tbcd/level" "github.com/hemilabs/heminetwork/service/deucalion" + "github.com/hemilabs/heminetwork/service/pprof" "github.com/hemilabs/heminetwork/ttl" ) @@ -162,6 +163,7 @@ type Config struct { MaxCachedTxs int Network string PrometheusListenAddress string + PprofListenAddress string } func NewDefaultConfig() *Config { @@ -1730,6 +1732,25 @@ func (s *Server) Run(pctx context.Context) error { log.Infof("RPC server shutdown cleanly") }() + // pprof + if s.cfg.PprofListenAddress != "" { + p, err := pprof.NewServer(&pprof.Config{ + ListenAddress: s.cfg.PprofListenAddress, + }) + if err != nil { + return fmt.Errorf("create pprof server: %w", err) + } + s.wg.Add(1) + go func() { + defer s.wg.Done() + if err := p.Run(ctx); !errors.Is(err, context.Canceled) { + log.Errorf("pprof server terminated with error: %v", err) + return + } + log.Infof("pprof server clean shutdown") + }() + } + // Prometheus if s.cfg.PrometheusListenAddress != "" { d, err := deucalion.New(&deucalion.Config{