Log prometheus listener error by default (#7284)

## What changed?
<!-- Describe what has changed in this PR -->
- Log error instead of panic for existing tally users when prometheus
listener can't be created.
- No behavior change for existing OTEL users

## Why?
<!-- Tell your future self why have you made these changes -->
- For backward compatibility

common/metrics/config.go

@@ -486,13 +486,15 @@ func MetricsHandlerFromConfig(logger log.Logger, c *Config) (Handler, error) {
 
 	setDefaultPerUnitHistogramBoundaries(&c.ClientConfig)
 
+	fatalOnListenerError := true
 	if c.Prometheus.Framework != FrameworkOpentelemetry {
 		c.ClientConfig.WithoutUnitSuffix = true
 		c.ClientConfig.WithoutCounterSuffix = true
 		c.ClientConfig.RecordTimerInSeconds = true
+		fatalOnListenerError = false
 	}
 
-	otelProvider, err := NewOpenTelemetryProvider(logger, c.Prometheus, &c.ClientConfig)
+	otelProvider, err := NewOpenTelemetryProvider(logger, c.Prometheus, &c.ClientConfig, fatalOnListenerError)
 	if err != nil {
 		logger.Fatal(err.Error())
 	}

common/metrics/opentelemetry_provider.go

@@ -57,6 +57,7 @@ func NewOpenTelemetryProvider(
 	logger log.Logger,
 	prometheusConfig *PrometheusConfig,
 	clientConfig *ClientConfig,
+	fatalOnListenerError bool,
 ) (*openTelemetryProviderImpl, error) {
 	reg := prometheus.NewRegistry()
 	exporterOpts := []exporters.Option{exporters.WithRegisterer(reg)}
@@ -91,7 +92,7 @@ func NewOpenTelemetryProvider(
 		sdkmetrics.WithReader(exporter),
 		sdkmetrics.WithView(views...),
 	)
-	metricServer := initPrometheusListener(prometheusConfig, reg, logger)
+	metricServer := initPrometheusListener(prometheusConfig, reg, logger, fatalOnListenerError)
 	meter := provider.Meter("temporal")
 	reporter := &openTelemetryProviderImpl{
 		meter:  meter,
@@ -102,7 +103,12 @@ func NewOpenTelemetryProvider(
 	return reporter, nil
 }
 
-func initPrometheusListener(config *PrometheusConfig, reg *prometheus.Registry, logger log.Logger) *http.Server {
+func initPrometheusListener(
+	config *PrometheusConfig,
+	reg *prometheus.Registry,
+	logger log.Logger,
+	fatalOnListenerError bool,
+) *http.Server {
 	handlerPath := config.HandlerPath
 	if handlerPath == "" {
 		handlerPath = "/metrics"
@@ -118,8 +124,15 @@ func initPrometheusListener(config *PrometheusConfig, reg *prometheus.Registry,
 
 	go func() {
 		err := server.ListenAndServe()
-		if err != http.ErrServerClosed {
-			logger.Fatal("Failed to initialize prometheus listener.", tag.Address(config.ListenAddress))
+		if err == http.ErrServerClosed {
+			return
+		}
+		msg := "Failed to initialize prometheus listener."
+		logger := log.With(logger, tag.Error(err), tag.Address(config.ListenAddress))
+		if fatalOnListenerError {
+			logger.Fatal(msg)
+		} else {
+			logger.Error(msg)
 		}
 	}()