Merge pull request #1490 from vikashsprem/feat/support-grpc

feat: support for both HTTP and gRPC protocols in meter/logging/tracing

internal/config/config.go

@@ -95,6 +95,7 @@ type (
 		Insecure bool     `mapstructure:"insecure"` // Connect to the collector using the HTTP scheme, instead of HTTPS.
 		URLPath  string   `mapstructure:"path"`     // Path for the log exporter, if not defined /v1/logs will be used
 		Headers  []string `mapstructure:"headers"`
+		Protocol string   `mapstructure:"protocol"` // Protocol for the log exporter, e.g., http, grpc
 	}
 
 	// Tracer contains configuration for distributed tracing.
@@ -105,6 +106,7 @@ type (
 		Insecure bool     `mapstructure:"insecure"` // Connect to the collector using the HTTP scheme, instead of HTTPS.
 		URLPath  string   `mapstructure:"path"`     // Path for the tracing exporter, if not defined /v1/trace will be used
 		Headers  []string `mapstructure:"headers"`
+		Protocol string   `mapstructure:"protocol"` // Protocol for the tracing exporter, e.g., http, grpc
 	}
 
 	// Meter contains configuration for metrics collection and reporting.
@@ -116,6 +118,7 @@ type (
 		URLPath  string   `mapstructure:"path"`     // Path for the metrics exporter, if not defined /v1/metrics will be used
 		Headers  []string `mapstructure:"headers"`
 		Interval int      `mapstructure:"interval"`
+		Protocol string   `mapstructure:"protocol"` // Protocol for the metrics exporter, e.g., http, grpc
 	}
 
 	// Service contains configuration for various service-level features.
@@ -295,17 +298,20 @@ func DefaultConfig() *Config {
 			Enabled:  false,
 			Exporter: "otlp",
 			Headers:  []string{},
+			Protocol: "http",
 		},
 		Tracer: Tracer{
-			Enabled: false,
-			Headers: []string{},
+			Enabled:  false,
+			Headers:  []string{},
+			Protocol: "http",
 		},
 		Meter: Meter{
 			Enabled:  false,
 			Exporter: "otlp",
 			Endpoint: "telemetry.permify.co",
 			Headers:  []string{},
 			Interval: 300,
+			Protocol: "http",
 		},
 		Service: Service{
 			CircuitBreaker: false,

pkg/cmd/config.go

@@ -49,6 +49,7 @@ func NewConfigCommand() *cobra.Command {
 	f.Bool("log-insecure", conf.Log.Insecure, "use https or http for logs")
 	f.String("log-urlpath", conf.Log.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("log-headers", conf.Log.Headers, "allows setting custom headers for the log exporter in key-value pairs")
+	f.String("log-protocol", conf.Log.Protocol, "allows setting the communication protocol for the log exporter, with options http or grpc")
 	f.Bool("authn-enabled", conf.Authn.Enabled, "enable server authentication")
 	f.String("authn-method", conf.Authn.Method, "server authentication method")
 	f.StringSlice("authn-preshared-keys", conf.Authn.Preshared.Keys, "preshared key/keys for server authentication")
@@ -65,13 +66,15 @@ func NewConfigCommand() *cobra.Command {
 	f.Bool("tracer-insecure", conf.Tracer.Insecure, "use https or http for tracer data, only used for otlp exporter or signoz")
 	f.String("tracer-urlpath", conf.Tracer.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("tracer-headers", conf.Tracer.Headers, "allows setting custom headers for the tracer exporter in key-value pairs")
+	f.String("tracer-protocol", conf.Tracer.Protocol, "allows setting the communication protocol for the tracer exporter, with options http or grpc")
 	f.Bool("meter-enabled", conf.Meter.Enabled, "switch option for metric")
 	f.String("meter-exporter", conf.Meter.Exporter, "can be; otlp. (integrated metric tools)")
 	f.String("meter-endpoint", conf.Meter.Endpoint, "export uri for metric data")
 	f.Bool("meter-insecure", conf.Meter.Insecure, "use https or http for metric data")
 	f.String("meter-urlpath", conf.Meter.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("meter-headers", conf.Meter.Headers, "allows setting custom headers for the metric exporter in key-value pairs")
 	f.Int("meter-interval", conf.Meter.Interval, "allows to set metrics to be pushed in certain time interval")
+	f.String("meter-protocol", conf.Meter.Protocol, "allows setting the communication protocol for the meter exporter, with options http or grpc")
 	f.Bool("service-circuit-breaker", conf.Service.CircuitBreaker, "switch option for service circuit breaker")
 	f.Bool("service-watch-enabled", conf.Service.Watch.Enabled, "switch option for watch service")
 	f.Int64("service-schema-cache-number-of-counters", conf.Service.Schema.Cache.NumberOfCounters, "schema service cache number of counters")
@@ -161,6 +164,7 @@ func conf() func(cmd *cobra.Command, args []string) error {
 			[]string{"logger.insecure", fmt.Sprintf("%v", cfg.Log.Insecure), getKeyOrigin(cmd, "log-insecure", "PERMIFY_LOG_INSECURE")},
 			[]string{"logger.urlpath", cfg.Log.URLPath, getKeyOrigin(cmd, "log-urlpath", "PERMIFY_LOG_URL_PATH")},
 			[]string{"logger.headers", fmt.Sprintf("%v", cfg.Log.Headers), getKeyOrigin(cmd, "log-headers", "PERMIFY_LOG_HEADERS")},
+			[]string{"logger.protocol", cfg.Log.Protocol, getKeyOrigin(cmd, "log-protocol", "PERMIFY_LOG_PROTOCOL")},
 			// AUTHN
 			[]string{"authn.enabled", fmt.Sprintf("%v", cfg.Authn.Enabled), getKeyOrigin(cmd, "authn-enabled", "PERMIFY_AUTHN_ENABLED")},
 			[]string{"authn.method", cfg.Authn.Method, getKeyOrigin(cmd, "authn-method", "PERMIFY_AUTHN_METHOD")},
@@ -179,13 +183,16 @@ func conf() func(cmd *cobra.Command, args []string) error {
 			[]string{"tracer.insecure", fmt.Sprintf("%v", cfg.Tracer.Insecure), getKeyOrigin(cmd, "tracer-insecure", "PERMIFY_TRACER_INSECURE")},
 			[]string{"tracer.urlpath", cfg.Tracer.URLPath, getKeyOrigin(cmd, "tracer-urlpath", "PERMIFY_TRACER_URL_PATH")},
 			[]string{"tracer.headers", fmt.Sprintf("%v", cfg.Tracer.Headers), getKeyOrigin(cmd, "tracer-headers", "PERMIFY_TRACER_HEADERS")},
+			[]string{"tracer.protocol", cfg.Tracer.Protocol, getKeyOrigin(cmd, "tracer-protocol", "PERMIFY_TRACER_PROTOCOL")},
 			// METER
 			[]string{"meter.enabled", fmt.Sprintf("%v", cfg.Meter.Enabled), getKeyOrigin(cmd, "meter-enabled", "PERMIFY_METER_ENABLED")},
 			[]string{"meter.exporter", cfg.Meter.Exporter, getKeyOrigin(cmd, "meter-exporter", "PERMIFY_METER_EXPORTER")},
 			[]string{"meter.endpoint", HideSecret(cfg.Meter.Exporter), getKeyOrigin(cmd, "meter-endpoint", "PERMIFY_METER_ENDPOINT")},
 			[]string{"meter.insecure", fmt.Sprintf("%v", cfg.Meter.Insecure), getKeyOrigin(cmd, "meter-insecure", "PERMIFY_METER_INSECURE")},
 			[]string{"meter.urlpath", cfg.Meter.URLPath, getKeyOrigin(cmd, "meter-urlpath", "PERMIFY_METER_URL_PATH")},
 			[]string{"meter.headers", fmt.Sprintf("%v", cfg.Meter.Headers), getKeyOrigin(cmd, "meter-headers", "PERMIFY_METER_HEADERS")},
+			[]string{"meter.protocol", cfg.Meter.Protocol, getKeyOrigin(cmd, "meter-protocol", "PERMIFY_METER_PROTOCOL")},
+			[]string{"meter.interval", fmt.Sprintf("%v", cfg.Meter.Interval), getKeyOrigin(cmd, "meter-interval", "PERMIFY_METER_INTERVAL")},
 			// SERVICE
 			[]string{"service.circuit_breaker", fmt.Sprintf("%v", cfg.Service.CircuitBreaker), getKeyOrigin(cmd, "service-circuit-breaker", "PERMIFY_SERVICE_CIRCUIT_BREAKER")},
 			[]string{"service.schema.cache.number_of_counters", fmt.Sprintf("%v", cfg.Service.Schema.Cache.NumberOfCounters), getKeyOrigin(cmd, "service-schema-cache-number-of-counters", "PERMIFY_SERVICE_WATCH_ENABLED")},

pkg/cmd/flags/serve.go

@@ -187,6 +187,13 @@ func RegisterServeFlags(flags *pflag.FlagSet) {
 		panic(err)
 	}
 
+	if err = viper.BindPFlag("logger.protocol", flags.Lookup("log-protocol")); err != nil {
+		panic(err)
+	}
+	if err = viper.BindEnv("logger.protocol", "PERMIFY_LOG_PROTOCOL"); err != nil {
+		panic(err)
+	}
+
 	// AUTHN
 	if err = viper.BindPFlag("authn.enabled", flags.Lookup("authn-enabled")); err != nil {
 		panic(err)
@@ -301,6 +308,13 @@ func RegisterServeFlags(flags *pflag.FlagSet) {
 		panic(err)
 	}
 
+	if err = viper.BindPFlag("tracer.protocol", flags.Lookup("tracer-protocol")); err != nil {
+		panic(err)
+	}
+	if err = viper.BindEnv("tracer.protocol", "PERMIFY_TRACER_PROTOCOL"); err != nil {
+		panic(err)
+	}
+
 	// METER
 	if err = viper.BindPFlag("meter.enabled", flags.Lookup("meter-enabled")); err != nil {
 		panic(err)
@@ -344,6 +358,20 @@ func RegisterServeFlags(flags *pflag.FlagSet) {
 		panic(err)
 	}
 
+	if err = viper.BindPFlag("meter.interval", flags.Lookup("meter-interval")); err != nil {
+		panic(err)
+	}
+	if err = viper.BindEnv("meter.interval", "PERMIFY_METER_INTERVAL"); err != nil {
+		panic(err)
+	}
+
+	if err = viper.BindPFlag("meter.protocol", flags.Lookup("meter-protocol")); err != nil {
+		panic(err)
+	}
+	if err = viper.BindEnv("meter.protocol", "PERMIFY_METER_PROTOCOL"); err != nil {
+		panic(err)
+	}
+
 	// SERVICE
 	if err = viper.BindPFlag("service.circuit_breaker", flags.Lookup("service-circuit-breaker")); err != nil {
 		panic(err)

pkg/cmd/serve.go

@@ -82,6 +82,7 @@ func NewServeCommand() *cobra.Command {
 	f.Bool("log-insecure", conf.Log.Insecure, "use https or http for logs")
 	f.String("log-urlpath", conf.Log.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("log-headers", conf.Log.Headers, "allows setting custom headers for the log exporter in key-value pairs")
+	f.String("log-protocol", conf.Log.Protocol, "allows setting the communication protocol for the log exporter, with options http or grpc")
 	f.Bool("authn-enabled", conf.Authn.Enabled, "enable server authentication")
 	f.String("authn-method", conf.Authn.Method, "server authentication method")
 	f.StringSlice("authn-preshared-keys", conf.Authn.Preshared.Keys, "preshared key/keys for server authentication")
@@ -98,12 +99,15 @@ func NewServeCommand() *cobra.Command {
 	f.Bool("tracer-insecure", conf.Tracer.Insecure, "use https or http for tracer data, only used for otlp exporter or signoz")
 	f.String("tracer-urlpath", conf.Tracer.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("tracer-headers", conf.Tracer.Headers, "allows setting custom headers for the tracer exporter in key-value pairs")
+	f.String("tracer-protocol", conf.Tracer.Protocol, "allows setting the communication protocol for the tracer exporter, with options http or grpc")
 	f.Bool("meter-enabled", conf.Meter.Enabled, "switch option for metric")
 	f.String("meter-exporter", conf.Meter.Exporter, "can be; otlp. (integrated metric tools)")
 	f.String("meter-endpoint", conf.Meter.Endpoint, "export uri for metric data")
 	f.Bool("meter-insecure", conf.Meter.Insecure, "use https or http for metric data")
 	f.String("meter-urlpath", conf.Meter.URLPath, "allow to set url path for otlp exporter")
 	f.StringSlice("meter-headers", conf.Meter.Headers, "allows setting custom headers for the metric exporter in key-value pairs")
+	f.Int("meter-interval", conf.Meter.Interval, "allows to set metrics to be pushed in certain time interval")
+	f.String("meter-protocol", conf.Meter.Protocol, "allows setting the communication protocol for the meter exporter, with options http or grpc")
 	f.Bool("service-circuit-breaker", conf.Service.CircuitBreaker, "switch option for service circuit breaker")
 	f.Bool("service-watch-enabled", conf.Service.Watch.Enabled, "switch option for watch service")
 	f.Int64("service-schema-cache-number-of-counters", conf.Service.Schema.Cache.NumberOfCounters, "schema service cache number of counters")
@@ -200,6 +204,7 @@ func serve() func(cmd *cobra.Command, args []string) error {
 				cfg.Log.Insecure,
 				cfg.Log.URLPath,
 				headers,
+				cfg.Log.Protocol,
 			)
 			if err != nil {
 				return errors.New("invalid logger exporter")
@@ -293,6 +298,7 @@ func serve() func(cmd *cobra.Command, args []string) error {
 				cfg.Tracer.Insecure,
 				cfg.Tracer.URLPath,
 				headers,
+				cfg.Tracer.Protocol,
 			)
 			if err != nil {
 				slog.Error(err.Error())
@@ -344,6 +350,7 @@ func serve() func(cmd *cobra.Command, args []string) error {
 				cfg.Meter.Insecure,
 				cfg.Meter.URLPath,
 				headers,
+				cfg.Meter.Protocol,
 			)
 
 			if err != nil {

pkg/telemetry/logexporters/factory.go

@@ -7,12 +7,10 @@ import (
 )
 
 // ExporterFactory - Create log exporter according to given params
-func ExporterFactory(name, endpoint string, insecure bool, urlpath string, headers map[string]string) (*otlplogs.Exporter, error) {
+func ExporterFactory(name, endpoint string, insecure bool, urlpath string, headers map[string]string, protocol string) (*otlplogs.Exporter, error) {
 	switch name {
-	case "otlp", "otlp-http":
-		return NewOTLP(endpoint, insecure, urlpath, headers)
-	case "otlp-grpc":
-		return nil, fmt.Errorf("%s log exporter is unsupported", name)
+	case "otlp", "otlp-http", "otlp-grpc":
+		return NewOTLP(endpoint, insecure, urlpath, headers, protocol)
 	default:
 		return nil, fmt.Errorf("%s log exporter is unsupported", name)
 	}

pkg/telemetry/logexporters/otlp.go

@@ -2,32 +2,58 @@ package logexporters
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs"
+	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogsgrpc"
 	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogshttp"
 )
 
-// NewOTLP - Creates new OTLP exporter using HTTP protocol.
-func NewOTLP(endpoint string, insecure bool, urlpath string, headers map[string]string) (*otlplogs.Exporter, error) {
-	options := []otlplogshttp.Option{
-		otlplogshttp.WithCompression(otlplogshttp.GzipCompression),
-		otlplogshttp.WithEndpoint(endpoint),
-	}
+// NewOTLP - Creates new OTLP exporter based on protocol.
+func NewOTLP(endpoint string, insecure bool, urlpath string, headers map[string]string, protocol string) (*otlplogs.Exporter, error) {
+	switch protocol {
+	case "http":
+		options := []otlplogshttp.Option{
+			otlplogshttp.WithCompression(otlplogshttp.GzipCompression),
+			otlplogshttp.WithEndpoint(endpoint),
+		}
 
-	if urlpath != "" {
-		options = append(options, otlplogshttp.WithURLPath(urlpath))
-	}
+		if urlpath != "" {
+			options = append(options, otlplogshttp.WithURLPath(urlpath))
+		}
 
-	if insecure {
-		options = append(options, otlplogshttp.WithInsecure())
-	}
+		if insecure {
+			options = append(options, otlplogshttp.WithInsecure())
+		}
 
-	exporter, err := otlplogs.NewExporter(context.Background(), otlplogs.WithClient(
-		otlplogshttp.NewClient(options...),
-	))
-	if err != nil {
-		return nil, err
-	}
+		exporter, err := otlplogs.NewExporter(context.Background(), otlplogs.WithClient(
+			otlplogshttp.NewClient(options...),
+		))
+		if err != nil {
+			return nil, err
+		}
+
+		return exporter, nil
 
-	return exporter, nil
+	case "grpc":
+		options := []otlplogsgrpc.Option{
+			otlplogsgrpc.WithEndpoint(endpoint),
+		}
+
+		if insecure {
+			options = append(options, otlplogsgrpc.WithInsecure())
+		}
+
+		exporter, err := otlplogs.NewExporter(context.Background(), otlplogs.WithClient(
+			otlplogsgrpc.NewClient(options...),
+		))
+		if err != nil {
+			return nil, err
+		}
+
+		return exporter, nil
+
+	default:
+		return nil, fmt.Errorf("unsupported protocol: %s", protocol)
+	}
 }

pkg/telemetry/meterexporters/factory.go

@@ -7,12 +7,10 @@ import (
 )
 
 // ExporterFactory - Create meter exporter according to given params
-func ExporterFactory(name, endpoint string, insecure bool, urlpath string, headers map[string]string) (metric.Exporter, error) {
+func ExporterFactory(name, endpoint string, insecure bool, urlpath string, headers map[string]string, protocol string) (metric.Exporter, error) {
 	switch name {
-	case "otlp", "otlp-http":
-		return NewOTLP(endpoint, insecure, urlpath, headers)
-	case "otlp-grpc":
-		return NewOTLPGrpc(endpoint, insecure, headers)
+	case "otlp", "otlp-http", "otlp-grpc":
+		return NewOTLP(endpoint, insecure, urlpath, headers, protocol)
 	default:
 		return nil, fmt.Errorf("%s meter exporter is unsupported", name)
 	}

pkg/telemetry/meterexporters/otlp.go

@@ -2,34 +2,62 @@ package meterexporters
 
 import (
 	"context"
+	"fmt"
 
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
 	"go.opentelemetry.io/otel/sdk/metric"
+	"google.golang.org/grpc/credentials"
 )
 
-// NewOTLP - Creates new OTLP exporter using HTTP protocol.
-func NewOTLP(endpoint string, insecure bool, urlpath string, headers map[string]string) (metric.Exporter, error) {
-	options := []otlpmetrichttp.Option{
-		otlpmetrichttp.WithCompression(otlpmetrichttp.GzipCompression),
-		otlpmetrichttp.WithEndpoint(endpoint),
+// NewOTLP - Creates new OTLP exporter based on protocol.
+func NewOTLP(endpoint string, insecure bool, urlpath string, headers map[string]string, protocol string) (metric.Exporter, error) {
+	switch protocol {
+	case "http":
+		options := []otlpmetrichttp.Option{
+			otlpmetrichttp.WithCompression(otlpmetrichttp.GzipCompression),
+			otlpmetrichttp.WithEndpoint(endpoint),
+		}
+
+		if len(headers) > 0 {
+			options = append(options, otlpmetrichttp.WithHeaders(headers))
+		}
+
+		if urlpath != "" {
+			options = append(options, otlpmetrichttp.WithURLPath(urlpath))
+		}
+
+		if insecure {
+			options = append(options, otlpmetrichttp.WithInsecure())
+		}
+
+		exporter, err := otlpmetrichttp.New(context.Background(), options...)
+		if err != nil {
+			return nil, err
+		}
+
+		return exporter, nil
+
+	case "grpc":
+		options := []otlpmetricgrpc.Option{
+			otlpmetricgrpc.WithEndpoint(endpoint),
+			otlpmetricgrpc.WithHeaders(headers),
+		}
+
+		if insecure {
+			options = append(options, otlpmetricgrpc.WithInsecure())
+		} else {
+			options = append(options, otlpmetricgrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, "")))
+		}
+
+		exporter, err := otlpmetricgrpc.New(context.Background(), options...)
+		if err != nil {
+			return nil, err
+		}
+
+		return exporter, nil
+
+	default:
+		return nil, fmt.Errorf("unsupported protocol: %s", protocol)
 	}
-
-	if len(headers) > 0 {
-		options = append(options, otlpmetrichttp.WithHeaders(headers))
-	}
-
-	if urlpath != "" {
-		options = append(options, otlpmetrichttp.WithURLPath(urlpath))
-	}
-
-	if insecure {
-		options = append(options, otlpmetrichttp.WithInsecure())
-	}
-
-	exporter, err := otlpmetrichttp.New(context.Background(), options...)
-	if err != nil {
-		return nil, err
-	}
-
-	return exporter, nil
 }