feat: read config from file

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Add support for speedscope rendering of Android reactnative profiles ([#386](https://github.com/getsentry/vroom/pull/386))
 - Add callTree generation for reactnative (android+js) profiles ([#390](https://github.com/getsentry/vroom/pull/390))
 - Use profiles that were not dynamically sampled to enhance slowest functions aggregation ([#300](https://github.com/getsentry/vroom/pull/300))
+- Add support for reading configuration values from file ([#383](https://github.com/getsentry/vroom/pull/383))
 
 **Bug Fixes**:
 

cmd/vroom/config.go

@@ -2,20 +2,30 @@ package main
 
 type (
 	ServiceConfig struct {
-		Environment string `env:"SENTRY_ENVIRONMENT" env-default:"development"`
-		Port        int    `env:"PORT"               env-default:"8085"`
+		Environment string `env:"SENTRY_ENVIRONMENT" yaml:"environment" env-default:"development"`
+		Host        string `env:"HOST" yaml:"host"`
+		Port        string `env:"PORT" yaml:"port" env-default:"8085"`
 
-		SentryDSN string `env:"SENTRY_DSN"`
+		SentryDSN string `env:"SENTRY_DSN" yaml:"sentry_dsn"`
 
-		OccurrencesKafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_OCCURRENCES" env-default:"localhost:9092"`
-		OccurrencesKafkaTopic   string   `env:"SENTRY_KAFKA_TOPIC_OCCURRENCES"   env-default:"ingest-occurrences"`
+		Occurrences struct {
+			KafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_OCCURRENCES" yaml:"kafka_brokers" env-default:"localhost:9092"`
+			KafkaTopic   string   `env:"SENTRY_KAFKA_TOPIC_OCCURRENCES"  yaml:"kafka_topic" env-default:"ingest-occurrences"`
+		} `yaml:"occurrences"`
 
-		ProfilingKafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_PROFILING" env-default:"localhost:9092"`
-		CallTreesKafkaTopic   string   `env:"SENTRY_KAFKA_TOPIC_CALL_TREES"  env-default:"profiles-call-tree"`
-		ProfilesKafkaTopic    string   `env:"SENTRY_KAKFA_TOPIC_PROFILES"    env-default:"processed-profiles"`
+		Profiling struct {
+			KafkaBrokers        []string `env:"SENTRY_KAFKA_BROKERS_PROFILING" yaml:"kafka_brokers" env-default:"localhost:9092"`
+			CallTreesKafkaTopic string   `env:"SENTRY_KAFKA_TOPIC_CALL_TREES" yaml:"call_trees_kafka_topic" env-default:"profiles-call-tree"`
+			ProfilesKafkaTopic  string   `env:"SENTRY_KAKFA_TOPIC_PROFILES" yaml:"profiles_kafka_topic" env-default:"processed-profiles"`
+		} `yaml:"profiling"`
 
-		SnubaHost string `env:"SENTRY_SNUBA_HOST" env-default:"http://localhost:1218"`
+		SnubaHost string `env:"SENTRY_SNUBA_HOST" yaml:"snuba_host" env-default:"http://localhost:1218"`
 
-		BucketURL string `env:"SENTRY_BUCKET_PROFILES" env-default:"file://./test/gcs/sentry-profiles"`
+		BucketURL string `env:"SENTRY_BUCKET_PROFILES" yaml:"bucket_url" env-default:"file://./test/gcs/sentry-profiles"`
+
+		Logging struct {
+			Level  string `env:"SENTRY_LOGGING_LEVEL" yaml:"level" env-default:"info"`
+			Format string `env:"SENTRY_LOGGING_FORMAT" yaml:"format" env-default:"simplified"`
+		} `yaml:"logging"`
 	}
 )

cmd/vroom/main.go

@@ -3,7 +3,10 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -14,6 +17,7 @@ import (
 	"github.com/CAFxX/httpcompression"
 	"github.com/getsentry/sentry-go"
 	sentryhttp "github.com/getsentry/sentry-go/http"
+	"github.com/getsentry/vroom/internal/logutil"
 	"github.com/ilyakaznacheev/cleanenv"
 	"github.com/julienschmidt/httprouter"
 	"github.com/rs/zerolog/log"
@@ -26,7 +30,6 @@ import (
 	"gocloud.dev/gcerrors"
 
 	"github.com/getsentry/vroom/internal/httputil"
-	"github.com/getsentry/vroom/internal/logutil"
 	"github.com/getsentry/vroom/internal/snubautil"
 )
 
@@ -48,11 +51,18 @@ const (
 	MiB       = 1024 * KiB
 )
 
-func newEnvironment() (*environment, error) {
+func newEnvironment(filePath string) (*environment, error) {
 	var e environment
-	err := cleanenv.ReadEnv(&e.config)
+	err := cleanenv.ReadConfig(filePath, &e.config)
 	if err != nil {
-		return nil, err
+		if errors.Is(err, os.ErrNotExist) {
+			err := cleanenv.ReadEnv(&e.config)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
 	}
 
 	e.snuba, err = snubautil.NewClient(e.config.SnubaHost, "profiles")
@@ -67,16 +77,16 @@ func newEnvironment() (*environment, error) {
 	}
 
 	e.occurrencesWriter = &kafka.Writer{
-		Addr:         kafka.TCP(e.config.OccurrencesKafkaBrokers...),
+		Addr:         kafka.TCP(e.config.Occurrences.KafkaBrokers...),
 		Async:        true,
 		Balancer:     kafka.CRC32Balancer{},
 		BatchSize:    100,
 		ReadTimeout:  3 * time.Second,
-		Topic:        e.config.OccurrencesKafkaTopic,
+		Topic:        e.config.Occurrences.KafkaTopic,
 		WriteTimeout: 3 * time.Second,
 	}
 	e.profilingWriter = &kafka.Writer{
-		Addr:         kafka.TCP(e.config.ProfilingKafkaBrokers...),
+		Addr:         kafka.TCP(e.config.Profiling.KafkaBrokers...),
 		Async:        true,
 		Balancer:     kafka.CRC32Balancer{},
 		BatchBytes:   20 * MiB,
@@ -165,13 +175,16 @@ func (e *environment) newRouter() (*httprouter.Router, error) {
 }
 
 func main() {
-	logutil.ConfigureLogger()
+	configFilePath := flag.String("c", "config.yml", "Path to configuration file")
+	flag.Parse()
 
-	env, err := newEnvironment()
+	env, err := newEnvironment(*configFilePath)
 	if err != nil {
 		log.Fatal().Err(err).Msg("error setting up environment")
 	}
 
+	logutil.ConfigureLogger(env.config.Logging.Level, env.config.Logging.Format)
+
 	err = sentry.Init(sentry.ClientOptions{
 		Dsn:                   env.config.SentryDSN,
 		EnableTracing:         true,
@@ -204,7 +217,7 @@ func main() {
 	}
 
 	server := http.Server{
-		Addr:              fmt.Sprintf(":%d", env.config.Port),
+		Addr:              net.JoinHostPort(env.config.Host, env.config.Port),
 		ReadHeaderTimeout: time.Second,
 		Handler:           sentryhttp.New(sentryhttp.Options{}).Handle(router),
 	}
@@ -227,7 +240,7 @@ func main() {
 	}()
 
 	err = server.ListenAndServe()
-	if err != nil && err != http.ErrServerClosed {
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		sentry.CaptureException(err)
 		log.Err(err).Msg("server failed")
 	}

cmd/vroom/profile.go

@@ -170,7 +170,7 @@ func (env *environment) postProfile(w http.ResponseWriter, r *http.Request) {
 		s = sentry.StartSpan(ctx, "processing")
 		s.Description = "Send functions to Kafka"
 		err = env.profilingWriter.WriteMessages(ctx, kafka.Message{
-			Topic: env.config.CallTreesKafkaTopic,
+			Topic: env.config.Profiling.CallTreesKafkaTopic,
 			Value: b,
 		})
 		s.Finish()
@@ -197,7 +197,7 @@ func (env *environment) postProfile(w http.ResponseWriter, r *http.Request) {
 		s = sentry.StartSpan(ctx, "processing")
 		s.Description = "Send profile metadata to Kafka"
 		err = env.profilingWriter.WriteMessages(ctx, kafka.Message{
-			Topic: env.config.ProfilesKafkaTopic,
+			Topic: env.config.Profiling.ProfilesKafkaTopic,
 			Value: b,
 		})
 		s.Finish()

config.example.yml

@@ -0,0 +1,49 @@
+# vroom binds to all host by default, if you want to bind to specific host,
+# uncomment the line below
+# host:
+port: 8085
+
+environment: "development"
+
+sentry_dsn: ""
+
+occurrences:
+  kafka_brokers:
+    - "localhost:9092"
+  kafka_topic: "ingest-occurrences"
+
+profiling:
+  kafka_brokers:
+    - "localhost:9092"
+  call_trees_kafka_topic: "profiles-call-tree"
+  profiles_kafka_topic: "processed-profiles"
+
+snuba_host: "http://localhost:1218"
+
+# The default bucket URL points to local filesystem. This bucket stores the profiles data compressed with LZ4.
+# For connecting to custom blob storage such as S3, GCS, or Azure Blob, change the value below.
+#
+# For S3: s3://my-bucket-name?region=us-west-1&awssdk=2&s3ForcePathStyle=false&disableSSL=true
+# For S3 hosted on MinIO: s3://my-bucket-name?awssdk=2&endpoint=minio.selfhosted.com&s3ForcePathStyle=true&disableSSL=true
+# Do provide these as environment variables (if needed):
+#   AWS_ACCESS_KEY=foobar
+#   AWS_SECRET_KEY=foobar
+#   AWS_SESSION_TOKEN=foobar (not required)
+#
+# For GCS: gs://my-bucket-name
+# Do provide these as environment variables (if needed):
+#   GOOGLE_APPLICATION_CREDENTIALS=foobar
+#
+# For Azure Blob: azblob://my-container?protocol=http&domain=localhost:10001&protocol=string&localemu=false&cdn=false
+# Do provide these as environment variables (if needed):
+#   AZURE_STORAGE_ACCOUNT=foobar
+#   AZURE_STORAGE_KEY=foobar
+#   AZURE_STORAGE_SAS_TOKEN=foobar
+bucket_url: "file://./test/gcs/sentry-profiles"
+
+logging:
+  # Available values are: "trace", "debug", "info", "warn", "error", "fatal", "panic", "disabled"
+  level: "info"
+  # "simplified" provides human-readable log on the console.
+  # "json" provides JSON log on the console
+  format: "simplified"

internal/logutil/logutil.go

@@ -9,13 +9,22 @@ import (
 	"cloud.google.com/go/compute/metadata"
 )
 
-func ConfigureLogger() {
+func ConfigureLogger(level string, format string) {
+	logLevel, err := zerolog.ParseLevel(level)
+	if err != nil {
+		logLevel = zerolog.WarnLevel
+	}
+
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 	log.Logger = log.With().Caller().Stack().Logger()
+	log.Logger = log.Sample(LevelSampler{Level: logLevel})
+
 	if metadata.OnGCE() {
 		log.Logger = log.Hook(ErrorHook{})
 	} else {
-		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+		if format == "simplified" {
+			log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+		}
 	}
 }
 

internal/logutil/sampler.go

@@ -0,0 +1,13 @@
+package logutil
+
+import (
+	"github.com/rs/zerolog"
+)
+
+type LevelSampler struct {
+	Level zerolog.Level
+}
+
+func (l LevelSampler) Sample(lvl zerolog.Level) bool {
+	return lvl >= l.Level
+}