Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add log encoding option for JSON logging. #2166

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Add log encoding option for JSON logging. #2166

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f2a81b9
Add log encoding option for JSON logging.
mcdee Sep 20, 2024
2dfc469
Merge branch 'main' into feature/log-encoding
mcdee Nov 11, 2024
0f8e343
Merge branch 'main' into feature/log-encoding
mcdee Nov 26, 2024
1ce0638
Linting.
mcdee Nov 26, 2024
7b5fba6
Merge branch 'main' into feature/log-encoding
mcdee Nov 27, 2024
d653ca6
Pass logger to NewWithOptions
mcdee Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions cmd/juno/juno.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ Juno is a Go implementation of a Starknet full-node client created by Nethermind
const (
configF = "config"
logLevelF = "log-level"
logEncodingF = "log-encoding"
httpF = "http"
httpHostF = "http-host"
httpPortF = "http-port"
Expand Down Expand Up @@ -126,6 +127,7 @@ const (

configFlagUsage = "The YAML configuration file."
logLevelFlagUsage = "Options: trace, debug, info, warn, error."
logEncodingFlagUsage = "The encoding of the log: console, json."
httpUsage = "Enables the HTTP RPC server on the default port and interface."
httpHostUsage = "The interface on which the HTTP RPC server will listen for requests."
httpPortUsage = "The port on which the HTTP server will listen for requests."
Expand Down Expand Up @@ -197,9 +199,11 @@ func main() {

config := new(node.Config)
cmd := NewCmd(config, func(cmd *cobra.Command, _ []string) error {
_, err := fmt.Fprintf(cmd.OutOrStdout(), greeting, Version)
if err != nil {
return err
if config.LogEncoding == utils.CONSOLE {
_, err := fmt.Fprintf(cmd.OutOrStdout(), greeting, Version)
if err != nil {
return err
}
}

n, err := node.New(config, Version)
Expand Down Expand Up @@ -308,12 +312,14 @@ func NewCmd(config *node.Config, run func(*cobra.Command, []string) error) *cobr
// For testing purposes, these variables cannot be declared outside the function because Cobra
// may mutate their values.
defaultLogLevel := utils.INFO
defaultLogEncoding := utils.CONSOLE
defaultNetwork := utils.Mainnet
defaultMaxVMs := 3 * runtime.GOMAXPROCS(0)
defaultCNUnverifiableRange := []int{} // Uint64Slice is not supported in Flags()

junoCmd.Flags().StringVar(&cfgFile, configF, defaultConfig, configFlagUsage)
junoCmd.Flags().Var(&defaultLogLevel, logLevelF, logLevelFlagUsage)
junoCmd.Flags().Var(&defaultLogEncoding, logEncodingF, logEncodingFlagUsage)
junoCmd.Flags().Bool(httpF, defaultHTTP, httpUsage)
junoCmd.Flags().String(httpHostF, defaulHost, httpHostUsage)
junoCmd.Flags().Uint16(httpPortF, defaultHTTPPort, httpPortUsage)
Expand Down
17 changes: 17 additions & 0 deletions cmd/juno/juno_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ func TestConfigPrecedence(t *testing.T) {
// checks on the config, those will be checked by the node implementation.
defaultHost := "localhost"
defaultLogLevel := utils.INFO
defaultLogEncoding := utils.CONSOLE
defaultHTTP := false
defaultHTTPPort := uint16(6060)
defaultWS := false
Expand Down Expand Up @@ -84,6 +85,7 @@ func TestConfigPrecedence(t *testing.T) {
},
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: utils.CONSOLE,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -115,6 +117,7 @@ func TestConfigPrecedence(t *testing.T) {
"custom network config file": {
cfgFile: true,
cfgFileContents: `log-level: debug
log-encoding: JSON
http-host: 0.0.0.0
http-port: 4576
db-path: /home/.juno
Expand All @@ -129,6 +132,7 @@ cn-unverifiable-range: [0,10]
`,
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: utils.JSON,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -161,6 +165,7 @@ cn-unverifiable-range: [0,10]
inputArgs: []string{""},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down Expand Up @@ -193,6 +198,7 @@ cn-unverifiable-range: [0,10]
inputArgs: []string{"--config", ""},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down Expand Up @@ -230,6 +236,7 @@ cn-unverifiable-range: [0,10]
cfgFileContents: "\n",
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down Expand Up @@ -269,6 +276,7 @@ pprof: true
`,
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -305,6 +313,7 @@ http-port: 4576
`,
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -340,6 +349,7 @@ http-port: 4576
},
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -375,6 +385,7 @@ http-port: 4576
},
expectedConfig: &node.Config{
LogLevel: utils.DEBUG,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -434,6 +445,7 @@ db-cache-size: 1024
},
expectedConfig: &node.Config{
LogLevel: utils.ERROR,
LogEncoding: defaultLogEncoding,
HTTP: true,
HTTPHost: "127.0.0.1",
HTTPPort: 4577,
Expand Down Expand Up @@ -472,6 +484,7 @@ network: sepolia
inputArgs: []string{"--db-path", "/home/flag/.juno"},
expectedConfig: &node.Config{
LogLevel: utils.WARN,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 4576,
Expand Down Expand Up @@ -506,6 +519,7 @@ network: sepolia
inputArgs: []string{"--db-path", "/home/flag/.juno", "--pprof"},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down Expand Up @@ -538,6 +552,7 @@ network: sepolia
env: []string{"JUNO_HTTP_PORT", "8080", "JUNO_WS", "true", "JUNO_HTTP_HOST", "0.0.0.0"},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: "0.0.0.0",
HTTPPort: 8080,
Expand Down Expand Up @@ -571,6 +586,7 @@ network: sepolia
inputArgs: []string{"--db-path", "/home/flag/.juno"},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down Expand Up @@ -604,6 +620,7 @@ network: sepolia
env: []string{"JUNO_DB_PATH", "/home/env/.juno", "JUNO_GW_API_KEY", "apikey"},
expectedConfig: &node.Config{
LogLevel: defaultLogLevel,
LogEncoding: defaultLogEncoding,
HTTP: defaultHTTP,
HTTPHost: defaultHost,
HTTPPort: defaultHTTPPort,
Expand Down
4 changes: 2 additions & 2 deletions db/pebble/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ func New(path string) (db.DB, error) {
return newPebble(path, nil)
}

func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, colouredLogger bool) (db.DB, error) {
func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, colouredLogger bool, logEncoding utils.LogEncoding) (db.DB, error) {
mcdee marked this conversation as resolved.
Show resolved Hide resolved
// Ensure that the specified cache size meets a minimum threshold.
cacheSizeMB = max(cacheSizeMB, minCacheSizeMB)

dbLog, err := utils.NewZapLogger(utils.ERROR, colouredLogger)
dbLog, err := utils.NewZapLogger(utils.ERROR, colouredLogger, logEncoding)
if err != nil {
return nil, fmt.Errorf("create DB logger: %w", err)
}
Expand Down
49 changes: 25 additions & 24 deletions node/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,28 +46,29 @@ const (

// Config is the top-level juno configuration.
type Config struct {
LogLevel utils.LogLevel `mapstructure:"log-level"`
HTTP bool `mapstructure:"http"`
HTTPHost string `mapstructure:"http-host"`
HTTPPort uint16 `mapstructure:"http-port"`
RPCCorsEnable bool `mapstructure:"rpc-cors-enable"`
Websocket bool `mapstructure:"ws"`
WebsocketHost string `mapstructure:"ws-host"`
WebsocketPort uint16 `mapstructure:"ws-port"`
GRPC bool `mapstructure:"grpc"`
GRPCHost string `mapstructure:"grpc-host"`
GRPCPort uint16 `mapstructure:"grpc-port"`
DatabasePath string `mapstructure:"db-path"`
Network utils.Network `mapstructure:"network"`
EthNode string `mapstructure:"eth-node"`
DisableL1Verification bool `mapstructure:"disable-l1-verification"`
Pprof bool `mapstructure:"pprof"`
PprofHost string `mapstructure:"pprof-host"`
PprofPort uint16 `mapstructure:"pprof-port"`
Colour bool `mapstructure:"colour"`
PendingPollInterval time.Duration `mapstructure:"pending-poll-interval"`
RemoteDB string `mapstructure:"remote-db"`
VersionedConstantsFile string `mapstructure:"versioned-constants-file"`
LogLevel utils.LogLevel `mapstructure:"log-level"`
LogEncoding utils.LogEncoding `mapstructure:"log-encoding"`
HTTP bool `mapstructure:"http"`
HTTPHost string `mapstructure:"http-host"`
HTTPPort uint16 `mapstructure:"http-port"`
RPCCorsEnable bool `mapstructure:"rpc-cors-enable"`
Websocket bool `mapstructure:"ws"`
WebsocketHost string `mapstructure:"ws-host"`
WebsocketPort uint16 `mapstructure:"ws-port"`
GRPC bool `mapstructure:"grpc"`
GRPCHost string `mapstructure:"grpc-host"`
GRPCPort uint16 `mapstructure:"grpc-port"`
DatabasePath string `mapstructure:"db-path"`
Network utils.Network `mapstructure:"network"`
EthNode string `mapstructure:"eth-node"`
DisableL1Verification bool `mapstructure:"disable-l1-verification"`
Pprof bool `mapstructure:"pprof"`
PprofHost string `mapstructure:"pprof-host"`
PprofPort uint16 `mapstructure:"pprof-port"`
Colour bool `mapstructure:"colour"`
PendingPollInterval time.Duration `mapstructure:"pending-poll-interval"`
RemoteDB string `mapstructure:"remote-db"`
VersionedConstantsFile string `mapstructure:"versioned-constants-file"`

Metrics bool `mapstructure:"metrics"`
MetricsHost string `mapstructure:"metrics-host"`
Expand Down Expand Up @@ -109,7 +110,7 @@ type Node struct {
// New sets the config and logger to the StarknetNode.
// Any errors while parsing the config on creating logger will be returned.
func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen
log, err := utils.NewZapLogger(cfg.LogLevel, cfg.Colour)
log, err := utils.NewZapLogger(cfg.LogLevel, cfg.Colour, cfg.LogEncoding)
if err != nil {
return nil, err
}
Expand All @@ -119,7 +120,7 @@ func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen
if dbIsRemote {
database, err = remote.New(cfg.RemoteDB, context.TODO(), log, grpc.WithTransportCredentials(insecure.NewCredentials()))
} else {
database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, cfg.Colour)
database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, cfg.Colour, cfg.LogEncoding)
}

if err != nil {
Expand Down
65 changes: 62 additions & 3 deletions utils/log.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,65 @@ func (l *LogLevel) UnmarshalText(text []byte) error {
return l.Set(string(text))
}

var ErrUnknownLogEncoding = fmt.Errorf(
"unknown log encoding (known: %s, %s)",
CONSOLE, JSON,
)

type LogEncoding int

// The following are necessary for Cobra and Viper, respectively, to unmarshal log level
// CLI/config parameters properly.
var (
_ pflag.Value = (*LogEncoding)(nil)
_ encoding.TextUnmarshaler = (*LogEncoding)(nil)
)

const (
CONSOLE LogEncoding = iota
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we rename it to Console please? according to standard go naming convention for const

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That isn't part of this PR, CONSOLE and some other constants in utils/log.go weren't created with this commit. I suggest a separate issue/PR if you want to address that.

JSON
)

func (l LogEncoding) String() string {
switch l {
case CONSOLE:
return "console"
case JSON:
return "json"
default:
// Should not happen.
panic(ErrUnknownLogEncoding)
}
}

func (l LogEncoding) MarshalYAML() (interface{}, error) {
return l.String(), nil
}

func (l *LogEncoding) Set(s string) error {
switch s {
case "CONSOLE", "console":
*l = CONSOLE
case "JSON", "json":
*l = JSON
default:
return ErrUnknownLogEncoding
}
return nil
}

func (l *LogEncoding) Type() string {
return "LogEncoding"
}

func (l *LogEncoding) MarshalText() ([]byte, error) {
return []byte(l.String()), nil
}

func (l *LogEncoding) UnmarshalText(text []byte) error {
return l.Set(string(text))
}

type Logger interface {
SimpleLogger
pebble.Logger
Expand Down Expand Up @@ -127,12 +186,12 @@ func NewNopZapLogger() *ZapLogger {
return &ZapLogger{zap.NewNop().Sugar()}
}

func NewZapLogger(logLevel LogLevel, colour bool) (*ZapLogger, error) {
func NewZapLogger(logLevel LogLevel, colour bool, encoding LogEncoding) (*ZapLogger, error) {
config := zap.NewProductionConfig()
config.Sampling = nil
config.Encoding = "console"
config.Encoding = encoding.String()
config.EncoderConfig.EncodeLevel = capitalColorLevelEncoder
if !colour {
if encoding == JSON || !colour {
config.EncoderConfig.EncodeLevel = capitalLevelEncoder
}
config.EncoderConfig.EncodeTime = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) {
Expand Down
4 changes: 2 additions & 2 deletions utils/log_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func TestLogLevelType(t *testing.T) {
func TestZapWithColour(t *testing.T) {
for level, str := range levelStrings {
t.Run("level: "+str, func(t *testing.T) {
_, err := utils.NewZapLogger(level, true)
_, err := utils.NewZapLogger(level, true, utils.CONSOLE)
assert.NoError(t, err)
})
}
Expand All @@ -102,7 +102,7 @@ func TestZapWithColour(t *testing.T) {
func TestZapWithoutColour(t *testing.T) {
for level, str := range levelStrings {
t.Run("level: "+str, func(t *testing.T) {
_, err := utils.NewZapLogger(level, false)
_, err := utils.NewZapLogger(level, false, utils.CONSOLE)
assert.NoError(t, err)
})
}
Expand Down
2 changes: 1 addition & 1 deletion vm/vm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func TestV1Call(t *testing.T) {
*classHash: simpleClass,
}))

log, err := utils.NewZapLogger(utils.ERROR, false)
log, err := utils.NewZapLogger(utils.ERROR, false, utils.CONSOLE)
require.NoError(t, err)

// test_storage_read
Expand Down