Merge branch 'main' into e2e-otlp

.github/actions/verify-metrics-snapshot/action.yaml

@@ -0,0 +1,70 @@
+# Copyright (c) 2023 The Jaeger Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+name: 'Verify Metric Snapshot and Upload Metrics'
+description: 'Upload or cache the metrics data after verification'
+inputs:
+  snapshot:
+    description: 'Path to the metric file'
+    required: true
+  artifact_key:
+    description: 'Artifact key used for uploading and fetching artifacts'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Upload current metrics snapshot
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: ${{ inputs.artifact_key }}
+        path: ./.metrics/${{ inputs.snapshot }}.txt
+        retention-days: 7
+
+      # The github cache restore successfully restores when cache saved has same key and same path.
+      # Hence to restore release metric with name relese_{metric_name} , the name must be changed to the same.
+    - name: Change file name before caching
+      if: github.ref_name == 'main'
+      shell: bash
+      run: |
+        mv ./.metrics/${{ inputs.snapshot }}.txt ./.metrics/baseline_${{ inputs.snapshot }}.txt
+
+    - name: Cache metrics snapshot on main branch for longer retention
+      if: github.ref_name == 'main'
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with: 
+        path: ./.metrics/baseline_${{ inputs.snapshot }}.txt
+        key: ${{ inputs.artifact_key }}_${{ github.run_id }}
+
+    # Use restore keys to match prefix and fetch the latest cache
+    # Here , restore keys is an ordered list of prefixes that need to be matched 
+    - name: Download the cached tagged metrics
+      id: download-release-snapshot
+      if: github.ref_name != 'main'
+      uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with:
+        path: ./.metrics/baseline_${{ inputs.snapshot }}.txt
+        key: ${{ inputs.artifact_key }}
+        restore-keys: |
+          ${{ inputs.artifact_key }}
+
+    - name: Calculate diff between the snapshots 
+      id: compare-snapshots
+      if: ${{ (github.ref_name != 'main') && (steps.download-release-snapshot.outputs.cache-matched-key != '')  }}
+      shell: bash
+      run: |
+        python3 -m pip install prometheus-client
+        python3 ./scripts/e2e/compare_metrics.py --file1 ./.metrics/${{ inputs.snapshot }}.txt --file2 ./.metrics/baseline_${{ inputs.snapshot }}.txt --output ./.metrics/diff_${{ inputs.snapshot }}.txt
+        if [ $? -eq 1 ]; then
+        echo "🛑 Differences found in metrics"
+        exit 1
+        fi
+
+    - name: Upload the diff artifact
+      if: ${{ (github.ref_name != 'main') && (steps.compare-snapshots.outcome == 'failure') }}
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: diff_${{ inputs.artifact_key }}
+        path: ./.metrics/diff_${{ inputs.snapshot }}.txt
+        retention-days: 7
+
+

.github/workflows/ci-e2e-all.yml

@@ -36,7 +36,3 @@ jobs:
   opensearch:
     uses: ./.github/workflows/ci-e2e-opensearch.yml
 
-
-
-
-

.github/workflows/ci-e2e-badger.yaml

@@ -25,7 +25,6 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -41,6 +40,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version == 'v2'
+      with:
+        snapshot: metrics_snapshot_badger
+        artifact_key: metrics_snapshot_badger_${{ matrix.version }}
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-cassandra.yml

@@ -48,9 +48,17 @@ jobs:
       run: bash scripts/e2e/cassandra.sh ${{ matrix.version.major }} ${{ matrix.version.schema }} ${{ matrix.jaeger-version }}
       env:
         SKIP_APPLY_SCHEMA: ${{ matrix.create-schema == 'auto' && true || false }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.jaeger-version == 'v2'
+      with:
+        snapshot: metrics_snapshot_cassandra
+        artifact_key: metrics_snapshot_cassandras_${{ matrix.version.major }}_${{ matrix.version.schema }}_${{ matrix.jaeger-version }}_${{ matrix.create-schema }}
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:
         files: cover.out
         flags: cassandra-${{ matrix.version.major }}-${{ matrix.jaeger-version }}-${{ matrix.create-schema }}
+
+

.github/workflows/ci-e2e-elasticsearch.yml

@@ -40,11 +40,7 @@ jobs:
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         submodules: true
-
-    - name: Fetch git tags
-      run: |
-        git fetch --prune --unshallow --tags
-
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -58,6 +54,12 @@ jobs:
     - name: Run ${{ matrix.version.distribution }} integration tests
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version.jaeger == 'v2'
+      with:
+        snapshot: metrics_snapshot_elasticsearch
+        artifact_key: metrics_snapshot_elasticsearch_${{ matrix.version.major }}_${{ matrix.version.jaeger}}
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-grpc.yml

@@ -41,6 +41,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version == 'v2'
+      with:
+        snapshot: metrics_snapshot_grpc
+        artifact_key: metrics_snapshot_grpc_${{ matrix.version }}
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-kafka.yml

@@ -35,6 +35,12 @@ jobs:
     - name: Run kafka integration tests
       id: test-execution
       run: bash scripts/e2e/kafka.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.jaeger-version == 'v2'
+      with:
+        snapshot: metrics_snapshot_kafka
+        artifact_key: metrics_snapshot_kafka_${{ matrix.jaeger-version }}
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-memory.yaml

@@ -21,15 +21,20 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
+        
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
 
     - name: Run Memory storage integration tests
       run: |
         STORAGE=memory_v2 make jaeger-v2-storage-integration-test
-
+        
+    - uses: ./.github/actions/verify-metrics-snapshot
+      with:
+        snapshot: metrics_snapshot_memory
+        artifact_key: metrics_snapshot_memory
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-opensearch.yml

@@ -38,10 +38,6 @@ jobs:
       with:
         submodules: true
 
-    - name: Fetch git tags
-      run: |
-        git fetch --prune --unshallow --tags
-
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -52,6 +48,12 @@ jobs:
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version.jaeger == 'v2'
+      with:
+        snapshot: metrics_snapshot_opensearch
+        artifact_key: metrics_snapshot_opensearch_${{ matrix.version.major }}
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-lint-checks.yaml

@@ -92,7 +92,7 @@ jobs:
       run: make thrift && git diff --name-status --exit-code
 
     - name: Verify Mockery types are up to date
-      run: make generate-mocks && git diff --name-status --exit-code
+      run: make generate-mocks && { if git status --porcelain | grep '??'; then exit 1; else git diff --name-status --exit-code; fi }
 
   lint-shell-scripts:
     runs-on: ubuntu-latest
@@ -155,7 +155,7 @@ jobs:
           jaeger_binary_size
 
     - name: Compare jaeger binary sizes
-      if: steps.cache-binary-size.outputs.cache-hit == 'true'
+      if: steps.cache-binary-size.outputs.cache-matched-key != ''
       run: |
         OLD_BINARY_SIZE=$(cat ./jaeger_binary_size.txt)
         NEW_BINARY_SIZE=$(cat ./new_jaeger_binary_size.txt)

.gitignore

@@ -50,3 +50,4 @@ sha256sum.combined.txt
 resource.syso
 .gocache
 test-results.json
+.metrics/

CONTRIBUTING.md

@@ -211,3 +211,16 @@ For example:
 ```
 * Remove deprecated flags `--old-flag`, please use `--new-flag` ([#1234](<pull-request URL>), [@myusername](https://github.com/myusername))
 ```
+
+## Using Feature Gates for Breaking Changes
+
+As much as possible, use OTel Collector's [feature gates][feature_gates] to manage breaking changes. For example, consider that we discovered a bug in the existing behavior, such as https://github.com/jaegertracing/jaeger/issues/5270. Simply changing the behavior might be a breaking change, so we implement a new behavior and create an internal config setting that enables or disables it. But how will users ever know and be encouraged to migrate to the new behavior? For that we can create a feature gate (without even creating any additional user-facing configuration), as follows:
+  * Introduce a new feature gate, with the name `jaeger.***`.
+  * If we don't want to change the default behavior right away, we can start the feature in the Alpha state, where it is disabled by default. No breaking changes need to be called out in the changelog.
+  * If we do want to change the default behavior right away, we can start the feature in the Beta state, where it is enabled by default, but the user can still disable it. Call out a breaking change in the changelog.
+  * Two releases later change the gate to Stable, where it is not only enabled by default, but trying to disable it will cause a runtime error. The code for the old behavior should be removed. Call out a breaking change in the changelog.
+  * Two releases later remove the feature gate as unused. Call out a breaking change in the changelog.
+
+See https://github.com/jaegertracing/jaeger/pull/6441 for an example of this workflow.
+
+[feature_gates]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/featuregate/README.md

Makefile.Protobuf.mk

@@ -91,11 +91,19 @@ proto-model:
 	$(call proto_compile, model, idl/proto/api_v2/model.proto)
 	$(PROTOC) -Imodel/proto --go_out=$(PWD)/model/ model/proto/model_test.proto
 
+API_V2_PATCHED_DIR=proto-gen/.patched/api_v2
+.PHONY: patch-api-v2
+patch-api-v2:
+	mkdir -p $(API_V2_PATCHED_DIR)
+	cp idl/proto/api_v2/collector.proto $(API_V2_PATCHED_DIR)/
+	cp idl/proto/api_v2/sampling.proto $(API_V2_PATCHED_DIR)/
+	cat idl/proto/api_v2/query.proto | $(SED) 's|jaegertracing/jaeger-idl/model/v1.|jaegertracing/jaeger/model.|g' > $(API_V2_PATCHED_DIR)/query.proto
+
 .PHONY: proto-api-v2
-proto-api-v2:
-	$(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/query.proto)
-	$(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/collector.proto)
-	$(call proto_compile, proto-gen/api_v2, idl/proto/api_v2/sampling.proto)
+proto-api-v2: patch-api-v2
+	$(call proto_compile, proto-gen/api_v2, $(API_V2_PATCHED_DIR)/query.proto)
+	$(call proto_compile, proto-gen/api_v2, $(API_V2_PATCHED_DIR)/collector.proto)
+	$(call proto_compile, proto-gen/api_v2, $(API_V2_PATCHED_DIR)/sampling.proto)
 
 .PHONY: proto-openmetrics
 proto-openmetrics:

README.md

@@ -136,6 +136,10 @@ Please see [ADOPTERS.md](./ADOPTERS.md) for some of the organizations using Jaeg
 If you would like to add your organization to the list, please comment on our
 [survey issue](https://github.com/jaegertracing/jaeger/issues/207).
 
+## Sponsors
+
+The Jaeger project owes its success in open source largely to the Cloud Native Computing Foundation (CNCF), our primary supporter. We deeply appreciate their vital support.  Furthermore, we are grateful to Uber for their initial, project-launching donation, and for the continuous contributions of software and infrastructure from 1Password, Codecov.io, Dosu, GitHub, Google Analytics, Netlify, and Oracle Cloud Infrastructure. Thank you for your generous support.
+
 ## License
 
 Copyright (c) The Jaeger Authors. [Apache 2.0 License](./LICENSE).

cmd/all-in-one/main.go

@@ -29,14 +29,14 @@ import (
 	queryApp "github.com/jaegertracing/jaeger/cmd/query/app"
 	"github.com/jaegertracing/jaeger/cmd/query/app/querysvc"
 	v2querysvc "github.com/jaegertracing/jaeger/cmd/query/app/querysvc/v2/querysvc"
+	ss "github.com/jaegertracing/jaeger/internal/sampling/samplingstrategy/metafactory"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/jtracer"
 	"github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/pkg/telemetry"
 	"github.com/jaegertracing/jaeger/pkg/tenancy"
 	"github.com/jaegertracing/jaeger/pkg/version"
 	"github.com/jaegertracing/jaeger/plugin/metricstore"
-	ss "github.com/jaegertracing/jaeger/plugin/sampling/strategyprovider"
 	"github.com/jaegertracing/jaeger/plugin/storage"
 	"github.com/jaegertracing/jaeger/ports"
 	"github.com/jaegertracing/jaeger/storage_v2/depstore"

cmd/collector/app/span_processor.go

@@ -227,8 +227,7 @@ func (sp *spanProcessor) saveSpan(span *model.Span, tenant string) {
 }
 
 func (sp *spanProcessor) writeSpan(ctx context.Context, span *model.Span) error {
-	spanWriter, err := v1adapter.GetV1Writer(sp.traceWriter)
-	if err == nil {
+	if spanWriter, ok := v1adapter.GetV1Writer(sp.traceWriter); ok {
 		return spanWriter.WriteSpan(ctx, span)
 	}
 	traces := v1adapter.V1BatchesToTraces([]*model.Batch{{Spans: []*model.Span{span}}})

cmd/collector/main.go

@@ -22,12 +22,12 @@ import (
 	cmdFlags "github.com/jaegertracing/jaeger/cmd/internal/flags"
 	"github.com/jaegertracing/jaeger/cmd/internal/printconfig"
 	"github.com/jaegertracing/jaeger/cmd/internal/status"
+	ss "github.com/jaegertracing/jaeger/internal/sampling/samplingstrategy/metafactory"
 	"github.com/jaegertracing/jaeger/pkg/config"
 	"github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/pkg/telemetry"
 	"github.com/jaegertracing/jaeger/pkg/tenancy"
 	"github.com/jaegertracing/jaeger/pkg/version"
-	ss "github.com/jaegertracing/jaeger/plugin/sampling/strategyprovider"
 	"github.com/jaegertracing/jaeger/plugin/storage"
 	"github.com/jaegertracing/jaeger/ports"
 	"github.com/jaegertracing/jaeger/storage_v2/v1adapter"

cmd/internal/env/command.go

@@ -10,8 +10,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	ss "github.com/jaegertracing/jaeger/internal/sampling/samplingstrategy/metafactory"
 	"github.com/jaegertracing/jaeger/plugin/metricstore"
-	"github.com/jaegertracing/jaeger/plugin/sampling/strategyprovider"
 	"github.com/jaegertracing/jaeger/plugin/storage"
 )
 
@@ -66,11 +66,11 @@ func Command() *cobra.Command {
 		"The type of backend used for service dependencies storage.",
 	)
 	fs.String(
-		strategyprovider.SamplingTypeEnvVar,
+		ss.SamplingTypeEnvVar,
 		"file",
 		fmt.Sprintf(
 			strings.ReplaceAll(samplingTypeDescription, "\n", " "),
-			strings.Join(strategyprovider.AllSamplingTypes, ", "),
+			strings.Join(ss.AllSamplingTypes, ", "),
 		),
 	)
 	fs.String(
@@ -79,7 +79,7 @@ func Command() *cobra.Command {
 		fmt.Sprintf(
 			strings.ReplaceAll(samplingStorageTypeDescription, "\n", " "),
 			strings.Join(storage.AllSamplingStorageTypes(), ", "),
-			strategyprovider.SamplingTypeEnvVar,
+			ss.SamplingTypeEnvVar,
 		),
 	)
 	fs.String(