diff --git a/.github/workflows/scripts/get-supported-version.sh b/.github/workflows/scripts/get-supported-version.sh new file mode 100755 index 00000000000..9c0b18fb8c4 --- /dev/null +++ b/.github/workflows/scripts/get-supported-version.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only + +set -o errexit +set -o nounset +set -o pipefail + +IMAGES_NAME="$1" +tags=$(skopeo --override-os linux inspect docker://$IMAGES_NAME | jq -r '.RepoTags' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" | sort -ur) +supported_versions=$(echo "$tags" | awk -F '.' '{print $1 "." $2}' | sort -ur | head -n 2) + +previous_major=$(echo "$tags" | awk -F '.' '{print $1}' | head -n 1) +previous_major=$((previous_major - 1)) + +previous_major_tags=$(echo "$tags" | grep -Eo "${previous_major}\.[0-9]+\.[0-9]+") +if [ -z "$previous_major_tags" ]; then + echo "No previous minor version found" +else + previous_minor=$(echo "$previous_major_tags" | awk -F '.' '{print $1 "." $2}' | sort -ur | head -n 1) + echo "Previous minor version is $previous_minor" +fi +# Construct the pattern for the last minor version of the previous major version +# pattern="^$previous_major\.[0-9]+$" +# | awk -F '.' '{print $1 "." $2}' +# Extract the last minor version of the previous major version +# last_minor_of_previous_major=$(echo "$latest_versions" | grep -E "$pattern" | sort -u | tail -n 1) + +# previous_major=$(echo "$current_major-1" | bc) + + +# latest_versions=$(printf "%s\n%s" "$latest_versions" "$previous_minor") +# Add prefix "release-" to each element +# latest_versions=$(echo "$latest_versions" | sed 's/^/release-/') + +echo "$previous_minor" diff --git a/cmd/mimir/main.go b/cmd/mimir/main.go index 923528b5765..1dd6c550dbf 100644 --- a/cmd/mimir/main.go +++ b/cmd/mimir/main.go @@ -22,11 +22,11 @@ import ( "github.com/grafana/dskit/flagext" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/common/tracing" "gopkg.in/yaml.v3" "github.com/grafana/mimir/pkg/mimir" util_log "github.com/grafana/mimir/pkg/util/log" + "github.com/grafana/mimir/pkg/util/trace" "github.com/grafana/mimir/pkg/util/usage" "github.com/grafana/mimir/pkg/util/version" ) @@ -180,12 +180,15 @@ func main() { } } - // Setting the environment variable JAEGER_AGENT_HOST enables tracing. - if trace, err := tracing.NewFromEnv(name); err != nil { - level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error()) - } else { - defer trace.Close() + if err := trace.InitTracingService(util_log.Logger); err != nil { + level.Error(util_log.Logger).Log("msg", "Failed to initialize tracing", "err", err.Error()) } + // Setting the environment variable JAEGER_AGENT_HOST enables tracing. + // if trace, err := tracing.NewFromEnv(name); err != nil { + // level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error()) + // } else { + // defer trace.Close() + // } } // Initialise seed for randomness usage. diff --git a/pkg/alertmanager/distributor.go b/pkg/alertmanager/distributor.go index 24651dd2fa1..8d72eb99362 100644 --- a/pkg/alertmanager/distributor.go +++ b/pkg/alertmanager/distributor.go @@ -20,7 +20,6 @@ import ( "github.com/grafana/dskit/ring" "github.com/grafana/dskit/ring/client" "github.com/grafana/dskit/services" - "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/weaveworks/common/httpgrpc" @@ -31,6 +30,7 @@ import ( "github.com/grafana/mimir/pkg/alertmanager/merger" "github.com/grafana/mimir/pkg/util" util_log "github.com/grafana/mimir/pkg/util/log" + "github.com/grafana/mimir/pkg/util/trace" ) // Distributor forwards requests to individual alertmanagers. @@ -175,8 +175,8 @@ func (d *Distributor) doQuorum(userID string, w http.ResponseWriter, r *http.Req err = ring.DoBatch(r.Context(), RingOp, d.alertmanagerRing, []uint32{shardByUser(userID)}, func(am ring.InstanceDesc, _ []int) error { // Use a background context to make sure all alertmanagers get the request even if we return early. localCtx := user.InjectOrgID(context.Background(), userID) - sp, localCtx := opentracing.StartSpanFromContext(localCtx, "Distributor.doQuorum") - defer sp.Finish() + localCtx, sp := trace.GetTracer().Start(localCtx, "Distributor.doQuorum") + defer sp.End() resp, err := d.doRequest(localCtx, am, &httpgrpc.HTTPRequest{ Method: r.Method, @@ -243,8 +243,8 @@ func (d *Distributor) doUnary(userID string, w http.ResponseWriter, r *http.Requ Headers: httpToHttpgrpcHeaders(r.Header), } - sp, ctx := opentracing.StartSpanFromContext(r.Context(), "Distributor.doUnary") - defer sp.Finish() + ctx, sp := trace.GetTracer().Start(r.Context(), "Distributor.doUnary") + defer sp.End() // Until we have a mechanism to combine the results from multiple alertmanagers, // we forward the request to only only of the alertmanagers. amDesc := replicationSet.Instances[rand.Intn(len(replicationSet.Instances))] diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 1c3142aaafe..b865d4f491f 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -34,6 +34,7 @@ import ( "github.com/weaveworks/common/instrument" "github.com/weaveworks/common/mtime" "github.com/weaveworks/common/user" + "go.opentelemetry.io/otel/attribute" "go.uber.org/atomic" "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" @@ -48,6 +49,7 @@ import ( util_math "github.com/grafana/mimir/pkg/util/math" "github.com/grafana/mimir/pkg/util/pool" "github.com/grafana/mimir/pkg/util/push" + "github.com/grafana/mimir/pkg/util/trace" "github.com/grafana/mimir/pkg/util/validation" ) @@ -707,9 +709,9 @@ func (d *Distributor) prePushHaDedupeMiddleware(next push.Func) push.Func { // Make a copy of these, since they may be retained as labels on our metrics, e.g. dedupedSamples. cluster, replica = copyString(cluster), copyString(replica) - span := opentracing.SpanFromContext(ctx) + ctx, span := trace.GetTracer().Start(ctx, "distributor.prePushHaDedupeMiddleware") if span != nil { - span.SetTag("cluster", cluster) + span.SetAttributes("cluster", cluster, attribute.String("cluster", cluster)) span.SetTag("replica", replica) } diff --git a/pkg/util/trace/openTelemetry.go b/pkg/util/trace/openTelemetry.go new file mode 100644 index 00000000000..80cc8018ddb --- /dev/null +++ b/pkg/util/trace/openTelemetry.go @@ -0,0 +1,76 @@ +package trace + +import ( + "context" + "net/http" + + "github.com/go-kit/log" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +type OpenTelemetry struct { + serviceName string + tracerProvider *tracesdk.TracerProvider + logger log.Logger +} + +type OpentelemetrySpan struct { + span trace.Span +} + +func (ote *OpenTelemetry) Inject(ctx context.Context, header http.Header, _ Span) { + otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(header)) +} + +func (ote *OpenTelemetry) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { + ctx, span := ote.tracerProvider.Tracer("").Start(ctx, spanName) + opentelemetrySpan := OpentelemetrySpan{ + span: span, + } + return ctx, opentelemetrySpan +} + +func (ote *OpenTelemetry) initTracerProvider() error { + // here we need to Integrate the PR in weaveworks/common here: https://github.com/weaveworks/common/pull/291/files + // or just copy this code for now and wait for people to review it + panic("implement me") +} + +func (s OpentelemetrySpan) AddEvents(keys []string, values []EventValue) { + for i, v := range values { + // TODO: add support for other types + if v.Num != 0 { + s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).String(v.Str))) + } + if v.Str != "" { + s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).Int64(v.Num))) + } + } +} + +func (s OpentelemetrySpan) End() { + s.span.End() +} + +func (s OpentelemetrySpan) RecordError(err error, options ...trace.EventOption) { + for _, o := range options { + s.span.RecordError(err, o) + } +} + +func (s OpentelemetrySpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) { + s.span.SetAttributes(kv) +} + +func (s OpentelemetrySpan) SetName(name string) { + s.span.SetName(name) +} + +func (s OpentelemetrySpan) SetStatus(code codes.Code, description string) { + s.span.SetStatus(code, description) +} diff --git a/pkg/util/trace/openTracing.go b/pkg/util/trace/openTracing.go new file mode 100644 index 00000000000..dfa2ec615c5 --- /dev/null +++ b/pkg/util/trace/openTracing.go @@ -0,0 +1,91 @@ +package trace + +import ( + "context" + "net/http" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" + otl "github.com/opentracing/opentracing-go/log" + "github.com/pkg/errors" + "github.com/weaveworks/common/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type OpenTracing struct { + serviceName string + logger log.Logger +} + +type OpenTracingSpan struct { + span opentracing.Span +} + +func (ot *OpenTracing) initTracerProvider() error { + if trace, err := tracing.NewFromEnv(ot.serviceName); err != nil { + return errors.Wrap(err, "failed to setup opentracing tracer") + } else { + defer trace.Close() + } + return nil +} + +func (ts *OpenTracing) Inject(ctx context.Context, header http.Header, span Span) { + opentracingSpan, ok := span.(OpenTracingSpan) + if !ok { + level.Error(ts.logger).Log("msg", "Failed to cast opentracing span") + } + err := opentracing.GlobalTracer().Inject( + opentracingSpan.span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(header)) + + if err != nil { + level.Error(ts.logger).Log("msg", "Failed to inject span context instance", "err", err) + } +} + +func (ts *OpenTracing) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) { + span, ctx := opentracing.StartSpanFromContext(ctx, spanName) + opentracingSpan := OpenTracingSpan{span: span} + return ctx, opentracingSpan +} + +func (s OpenTracingSpan) AddEvents(keys []string, values []EventValue) { + fields := []otl.Field{} + for i, v := range values { + if v.Str != "" { + field := otl.String(keys[i], v.Str) + fields = append(fields, field) + } + if v.Num != 0 { + field := otl.Int64(keys[i], v.Num) + fields = append(fields, field) + } + } + s.span.LogFields(fields...) +} + +func (s OpenTracingSpan) End() { + s.span.Finish() +} + +func (s OpenTracingSpan) RecordError(err error, options ...trace.EventOption) { + ext.Error.Set(s.span, true) +} + +func (s OpenTracingSpan) SetAttributes(key string, value interface{}, kv attribute.KeyValue) { + s.span.SetTag(key, value) +} + +func (s OpenTracingSpan) SetName(name string) { + s.span.SetOperationName(name) +} + +func (s OpenTracingSpan) SetStatus(code codes.Code, description string) { + ext.Error.Set(s.span, true) +} diff --git a/pkg/util/trace/trace.go b/pkg/util/trace/trace.go new file mode 100644 index 00000000000..45403fc8c9d --- /dev/null +++ b/pkg/util/trace/trace.go @@ -0,0 +1,99 @@ +package trace + +import ( + "context" + "net/http" + "os" + "strconv" + + "github.com/go-kit/log" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +var mimirTracer Tracer + +const ( + otelTracer = "OtelTracer" + serviceName = "mimir" +) + +type EventValue struct { + Str string + Num int64 +} + +// Tracer defines the service used to create new spans. +type Tracer interface { + // Start creates a new [Span] and places trace metadata on the + // [context.Context] passed to the method. + // Chose a low cardinality spanName and use [Span.SetAttributes] + // or [Span.AddEvents] for high cardinality data. + Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) + // Inject adds identifying information for the span to the + // headers defined in [http.Header] map (this mutates http.Header). + // + // Implementation quirk: Where OpenTelemetry is used, the [Span] is + // picked up from [context.Context] and for OpenTracing the + // information passed as [Span] is preferred. + Inject(context.Context, http.Header, Span) + + initTracerProvider() error +} + +type Span interface { + // End finalizes the Span and adds its end timestamp. + // Any further operations on the Span are not permitted after + // End has been called. + End() + // SetAttributes adds additional data to a span. + // SetAttributes repeats the key value pair with [string] and [any] + // used for OpenTracing and [attribute.KeyValue] used for + // OpenTelemetry. + SetAttributes(key string, value interface{}, kv attribute.KeyValue) + // SetName renames the span. + SetName(name string) + // SetStatus can be used to indicate whether the span was + // successfully or unsuccessfully executed. + // + // Only useful for OpenTelemetry. + SetStatus(code codes.Code, description string) + // RecordError adds an error to the span. + // + // Only useful for OpenTelemetry. + RecordError(err error, options ...trace.EventOption) + // AddEvents adds additional data with a temporal dimension to the + // span. + // + // Panics if the length of keys is shorter than the length of values. + AddEvents(keys []string, values []EventValue) +} + +// Opentracing is the default setting, but can be overridden by setting +func InitTracingService(logger log.Logger) error { + if e := os.Getenv(otelTracer); e != "" { + if value, err := strconv.ParseBool(e); err == nil { + if value { + mimirTracer = &OpenTelemetry{ + serviceName: serviceName, + logger: logger, + } + return mimirTracer.initTracerProvider() + } + } else { + return errors.Wrapf(err, "cannot parse env var %s=%s", otelTracer, e) + } + } + mimirTracer = &OpenTracing{ + serviceName: serviceName, + logger: logger, + } + return mimirTracer.initTracerProvider() +} + +// GetTracer returns the Tracer used by the service. +func GetTracer() Tracer { + return mimirTracer +}