merge with upstream v2.60.5 (#473)

* merge with upstream v2.60.5

* review comments

.github/workflows/qa-tip-tracking-gnosis.yml

@@ -0,0 +1,125 @@
+name: QA - Tip tracking (Gnosis)
+
+on:
+  push:
+    branches:
+      - 'release/2.*'
+  pull_request:
+    branches:
+      - 'release/2.*'
+    types:
+      - ready_for_review
+  workflow_dispatch:     # Run manually
+
+jobs:
+  tip-tracking-test:
+    runs-on: [self-hosted, Gnosis]
+    timeout-minutes: 600
+    env:
+      ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
+      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
+      ERIGON_QA_PATH: /home/qarunner/erigon-qa
+      TRACKING_TIME_SECONDS: 14400 # 4 hours
+      TOTAL_TIME_SECONDS: 28800 # 8 hours
+      CHAIN: gnosis
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+
+    - name: Clean Erigon Build Directory
+      run: |
+        make clean
+
+    - name: Build Erigon
+      run: |
+        make erigon
+      working-directory: ${{ github.workspace }}
+
+    - name: Pause the Erigon instance dedicated to db maintenance
+      run: |
+        python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
+
+    - name: Restore Erigon Testbed Data Directory
+      run: |
+        rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
+
+    - name: Run Erigon, wait sync and check ability to maintain sync
+      id: test_step
+      run: |
+        set +e # Disable exit on error
+        
+        # 1. Launch the testbed Erigon instance
+        # 2. Allow time for the Erigon to achieve synchronization
+        # 3. Begin timing the duration that Erigon maintains synchronization
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
+          ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon2 $CHAIN
+  
+        # Capture monitoring script exit status
+        test_exit_status=$?
+
+        # Save the subsection reached status
+        echo "::set-output name=test_executed::true"
+
+        # Clean up Erigon process if it's still running
+        if kill -0 $ERIGON_PID 2> /dev/null; then
+          echo "Terminating Erigon"
+          kill $ERIGON_PID
+          wait $ERIGON_PID
+        fi
+
+        # Check test runner script exit status
+        if [ $test_exit_status -eq 0 ]; then
+          echo "Tests completed successfully"
+          echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
+        else
+          echo "Error detected during tests"
+          echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Delete Erigon Testbed Data Directory
+      if: always()
+      run: |
+        rm -rf $ERIGON_TESTBED_DATA_DIR
+
+    - name: Resume the Erigon instance dedicated to db maintenance
+      run: |
+        python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
+
+    - name: Save test results
+      if: steps.test_step.outputs.test_executed == 'true'
+      env:
+        TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }}
+      run: |
+        db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit)
+        if [ -z "$db_version" ]; then
+          db_version="no-version"
+        fi
+        
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
+          --repo erigon \
+          --commit $(git rev-parse HEAD) \
+          --branch ${{ github.ref_name }} \
+          --test_name tip-tracking \
+          --chain $CHAIN \
+          --runner ${{ runner.name }} \
+          --db_version $db_version \
+          --outcome $TEST_RESULT \
+          --result_file ${{ github.workspace }}/result-$CHAIN.json
+
+    - name: Upload test results
+      if: steps.test_step.outputs.test_executed == 'true'
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results
+        path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+
+    - name: Action for Success
+      if: steps.test_step.outputs.TEST_RESULT == 'success'
+      run: echo "::notice::Tests completed successfully"
+
+    - name: Action for Not Success
+      if: steps.test_step.outputs.TEST_RESULT != 'success'
+      run: |
+        echo "::error::Error detected during tests"
+        exit 1

.goreleaser.yml

@@ -63,10 +63,4 @@ builds:
 
 
 snapshot:
-  name_template: "{{ .Tag }}.next"
-
-announce:
-  slack:
-    enabled: false
-    # The name of the channel that the user selected as a destination for webhook messages.
-    channel: '#code-releases'
+  name_template: "{{ .Tag }}.next"

Makefile

@@ -309,7 +309,6 @@ release: git-submodules
 		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
 		--clean --skip-validate
 
-
 # since DOCKER_UID, DOCKER_GID are default initialized to the current user uid/gid,
 # we need separate envvars to facilitate creation of the erigon user on the host OS.
 ERIGON_USER_UID ?= 3473

cl/beacon/beaconhttp/api.go

@@ -97,41 +97,41 @@ func HandleEndpoint[T any](h EndpointHandler[T]) http.HandlerFunc {
 		}
 		// TODO: potentially add a context option to buffer these
 		contentType := r.Header.Get("Accept")
-		contentTypes := strings.Split(contentType, ",")
 
 		// early return for event stream
 		if slices.Contains(w.Header().Values("Content-Type"), "text/event-stream") {
 			return
 		}
 		switch {
-		case slices.Contains(contentTypes, "application/octet-stream"):
+		case contentType == "*/*", contentType == "", strings.Contains(contentType, "text/html"), strings.Contains(contentType, "application/json"):
+			if !isNil(ans) {
+				w.Header().Set("Content-Type", "application/json")
+				err := json.NewEncoder(w).Encode(ans)
+				if err != nil {
+					// this error is fatal, log to console
+					log.Error("beaconapi failed to encode json", "type", reflect.TypeOf(ans), "err", err)
+				}
+			} else {
+				w.WriteHeader(200)
+			}
+		case strings.Contains(contentType, "application/octet-stream"):
 			sszMarshaler, ok := any(ans).(ssz.Marshaler)
 			if !ok {
 				NewEndpointError(http.StatusBadRequest, ErrorSszNotSupported).WriteTo(w)
 				return
 			}
+			w.Header().Set("Content-Type", "application/octet-stream")
 			// TODO: we should probably figure out some way to stream this in the future :)
 			encoded, err := sszMarshaler.EncodeSSZ(nil)
 			if err != nil {
 				WrapEndpointError(err).WriteTo(w)
 				return
 			}
 			w.Write(encoded)
-		case contentType == "*/*", contentType == "", slices.Contains(contentTypes, "text/html"), slices.Contains(contentTypes, "application/json"):
-			if !isNil(ans) {
-				w.Header().Add("content-type", "application/json")
-				err := json.NewEncoder(w).Encode(ans)
-				if err != nil {
-					// this error is fatal, log to console
-					log.Error("beaconapi failed to encode json", "type", reflect.TypeOf(ans), "err", err)
-				}
-			} else {
-				w.WriteHeader(200)
-			}
-		case slices.Contains(contentTypes, "text/event-stream"):
+		case strings.Contains(contentType, "text/event-stream"):
 			return
 		default:
-			http.Error(w, "content type must be application/json, application/octet-stream, or text/event-stream", http.StatusBadRequest)
+			http.Error(w, fmt.Sprintf("content type must include application/json, application/octet-stream, or text/event-stream, got %s", contentType), http.StatusBadRequest)
 		}
 	})
 }

cl/beacon/handler/states.go

@@ -205,9 +205,9 @@ func (a *ApiHandler) getFullState(w http.ResponseWriter, r *http.Request) (*beac
 }
 
 type finalityCheckpointsResponse struct {
-	FinalizedCheckpoint         solid.Checkpoint `json:"finalized_checkpoint"`
-	CurrentJustifiedCheckpoint  solid.Checkpoint `json:"current_justified_checkpoint"`
-	PreviousJustifiedCheckpoint solid.Checkpoint `json:"previous_justified_checkpoint"`
+	FinalizedCheckpoint         solid.Checkpoint `json:"finalized"`
+	CurrentJustifiedCheckpoint  solid.Checkpoint `json:"current_justified"`
+	PreviousJustifiedCheckpoint solid.Checkpoint `json:"previous_justified"`
 }
 
 func (a *ApiHandler) getFinalityCheckpoints(w http.ResponseWriter, r *http.Request) (*beaconhttp.BeaconResponse, error) {

cl/beacon/handler/states_test.go

@@ -426,7 +426,7 @@ func TestGetStateFinalityCheckpoints(t *testing.T) {
 			code:    http.StatusOK,
 		},
 	}
-	expected := `{"data":{"finalized_checkpoint":{"epoch":"1","root":"0xde46b0f2ed5e72f0cec20246403b14c963ec995d7c2825f3532b0460c09d5693"},"current_justified_checkpoint":{"epoch":"3","root":"0xa6e47f164b1a3ca30ea3b2144bd14711de442f51e5b634750a12a1734e24c987"},"previous_justified_checkpoint":{"epoch":"2","root":"0x4c3ee7969e485696669498a88c17f70e6999c40603e2f4338869004392069063"}},"execution_optimistic":false,"finalized":false,"version":"bellatrix"}` + "\n"
+	expected := `{"data":{"finalized":{"epoch":"1","root":"0xde46b0f2ed5e72f0cec20246403b14c963ec995d7c2825f3532b0460c09d5693"},"current_justified":{"epoch":"3","root":"0xa6e47f164b1a3ca30ea3b2144bd14711de442f51e5b634750a12a1734e24c987"},"previous_justified":{"epoch":"2","root":"0x4c3ee7969e485696669498a88c17f70e6999c40603e2f4338869004392069063"}},"execution_optimistic":false,"finalized":false,"version":"bellatrix"}` + "\n"
 	for _, c := range cases {
 		t.Run(c.blockID, func(t *testing.T) {
 			server := httptest.NewServer(handler.mux)

cl/beacon/handler/test_data/states_1.yaml

@@ -1,2 +1,2 @@
-finality_checkpoint: {"data":{"finalized_checkpoint":{"epoch":"1","root":"0xde46b0f2ed5e72f0cec20246403b14c963ec995d7c2825f3532b0460c09d5693"},"current_justified_checkpoint":{"epoch":"3","root":"0xa6e47f164b1a3ca30ea3b2144bd14711de442f51e5b634750a12a1734e24c987"},"previous_justified_checkpoint":{"epoch":"2","root":"0x4c3ee7969e485696669498a88c17f70e6999c40603e2f4338869004392069063"}},"finalized":false,"version":2,"execution_optimistic":false}
+finality_checkpoint: {"data":{"finalized":{"epoch":"1","root":"0xde46b0f2ed5e72f0cec20246403b14c963ec995d7c2825f3532b0460c09d5693"},"current_justified":{"epoch":"3","root":"0xa6e47f164b1a3ca30ea3b2144bd14711de442f51e5b634750a12a1734e24c987"},"previous_justified":{"epoch":"2","root":"0x4c3ee7969e485696669498a88c17f70e6999c40603e2f4338869004392069063"}},"finalized":false,"version":2,"execution_optimistic":false}
 randao: {"data":{"randao":"0xdeec617717272914bfd73e02ca1da113a83cf4cf33cd4939486509e2da4ccf4e"},"finalized":false,"execution_optimistic":false}

cl/clstages/clstages.go

@@ -32,14 +32,12 @@ func (s *StageGraph[CONFIG, ARGUMENTS]) StartWithStage(ctx context.Context, star
 		errch := make(chan error)
 		start := time.Now()
 		go func() {
-			sctx, cn := context.WithCancel(ctx)
-			defer cn()
 			// we run this is a goroutine so that the process can exit in the middle of a stage
 			// since caplin is designed to always be able to recover regardless of db state, this should be safe
 			select {
-			case errch <- currentStage.ActionFunc(sctx, lg, cfg, args):
-			case <-sctx.Done():
-				errch <- sctx.Err()
+			case errch <- currentStage.ActionFunc(ctx, lg, cfg, args):
+			case <-ctx.Done(): // we are not sure if actionFunc exits on ctx
+				errch <- ctx.Err()
 			}
 		}()
 		err := <-errch

cl/phase1/network/services/blob_sidecar_service.go

@@ -78,11 +78,15 @@ func (b *blobSidecarService) ProcessMessage(ctx context.Context, subnetId *uint6
 	currentSlot := b.ethClock.GetCurrentSlot()
 	sidecarSlot := msg.SignedBlockHeader.Header.Slot
 	// [IGNORE] The block is not from a future slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- i.e. validate that
-	//signed_beacon_block.message.slot <= current_slot (a client MAY queue future blocks for processing at the appropriate slot).
+	// signed_beacon_block.message.slot <= current_slot (a client MAY queue future blocks for processing at the appropriate slot).
 	if currentSlot < sidecarSlot && !b.ethClock.IsSlotCurrentSlotWithMaximumClockDisparity(sidecarSlot) {
 		return ErrIgnore
 	}
 
+	if b.forkchoiceStore.FinalizedSlot() >= sidecarSlot {
+		return ErrIgnore
+	}
+
 	blockRoot, err := msg.SignedBlockHeader.Header.HashSSZ()
 	if err != nil {
 		return err

cl/phase1/stages/clstages.go

@@ -252,7 +252,7 @@ func ConsensusClStages(ctx context.Context,
 					}
 					// This stage is special so use context.Background() TODO(Giulio2002): make the context be passed in
 					startingSlot := cfg.state.LatestBlockHeader().Slot
-					downloader := network2.NewBackwardBeaconDownloader(context.Background(), cfg.rpc, cfg.executionClient, cfg.indiciesDB)
+					downloader := network2.NewBackwardBeaconDownloader(ctx, cfg.rpc, cfg.executionClient, cfg.indiciesDB)
 
 					if err := SpawnStageHistoryDownload(StageHistoryReconstruction(downloader, cfg.antiquary, cfg.sn, cfg.indiciesDB, cfg.executionClient, cfg.beaconCfg, cfg.backfilling, cfg.blobBackfilling, false, startingRoot, startingSlot, cfg.tmpdir, 600*time.Millisecond, cfg.blockCollector, cfg.blockReader, cfg.blobStore, logger), context.Background(), logger); err != nil {
 						cfg.hasDownloaded = false