Merge branch 'master' into jord/sync-master-efm-jan14

Outstanding conflicts:
- consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go

.github/workflows/builds.yml

@@ -43,6 +43,10 @@ on:
         type: boolean
         description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
+      private_build:
+        type: boolean
+        description: 'Build private images'
+        required: false
 
 jobs:
   # matrix_builder generates a matrix that includes the roles selected in the input
@@ -82,6 +86,7 @@ jobs:
   docker-push:
     name: ${{ matrix.role }} images
     runs-on: ubuntu-latest
+    environment: Production Docker Registry
     needs: matrix_builder
 
     # setup jobs for each role
@@ -108,16 +113,30 @@ jobs:
         credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
     - name: Set up Google Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
-    - name: Authenticate docker with gcloud
+
+    - name: Authenticate Docker with gcloud
       run: |
-        gcloud auth configure-docker
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          gcloud auth configure-docker us-central1-docker.pkg.dev
+        else
+          gcloud auth configure-docker
+        fi
+
+    - name: Set CONTAINER_REGISTRY
+      id: set-registry
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV
+        else
+          echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV
+        fi
 
     - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
     - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
       if: ${{ inputs.include_alternative_builds }}
@@ -127,7 +146,7 @@ jobs:
       run: |
         make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
           docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
-          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
 
         

.github/workflows/cd.yml

@@ -1,10 +1,18 @@
 name: CD
 
 on:
-  push:
-    tags:
-      - '*'
-      - "!daily-*"
+# Workflow dispatch for now, while we're testing environments
+  # push:
+  #   tags:
+  #     - '*'
+  #     - "!daily-*"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag/commit'
+        required: true
+        type: string
+
 
 env:
   GO_VERSION: "1.22"
@@ -13,6 +21,7 @@ jobs:
   docker-push:
     name: Push to container registry
     runs-on: ubuntu-latest
+    environment: Production Docker Registry
     steps:
     - name: Setup Go
       uses: actions/setup-go@v4
@@ -21,6 +30,8 @@ jobs:
         go-version: ${{ env.GO_VERSION }}
     - name: Checkout repo
       uses: actions/checkout@v2
+      with:
+        ref: ${{ inputs.tag }}
     # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
     # Logging in as [email protected]
     - id: auth

.github/workflows/ci.yml

@@ -55,13 +55,19 @@ jobs:
         # https://github.com/golangci/golangci-lint-action/issues/244
         skip-cache: true
 
-
   tidy:
     name: Tidy
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -139,6 +145,13 @@ jobs:
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
+
+    - name: Setup private build environment
+      if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+      uses: ./actions/private-setup
+      with:
+        cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
     - name: Setup Go
       uses: actions/setup-go@v4
       timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -178,6 +191,13 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -216,6 +236,13 @@ jobs:
         with:
           # all tags are needed for integration tests
           fetch-depth: 0
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -255,9 +282,17 @@ jobs:
         targets: ${{ fromJSON(needs.create-integration-dynamic-test-matrix.outputs.dynamic-matrix)}}
     ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
     runs-on: ${{ matrix.targets.runner }}
+
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -301,6 +336,9 @@ jobs:
           - name: Access Cohort3 Integration Tests
             make: make -C integration access-cohort3-tests
             runner: ubuntu-latest
+          - name: Access Cohort4 Integration Tests
+            make: make -C integration access-cohort4-tests
+            runner: ubuntu-latest
             # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed
 #          - name: BFT (Framework) Integration Tests
 #            make: make -C integration bft-framework-tests
@@ -348,6 +386,13 @@ jobs:
       with:
           # all tags are needed for integration tests
           fetch-depth: 0
+
+    - name: Setup private build environment
+      if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+      uses: ./actions/private-setup
+      with:
+        cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
     - name: Setup Go
       uses: actions/setup-go@v4
       timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time

.golangci.yml

@@ -5,7 +5,7 @@ linters-settings:
   goimports:
     # put imports beginning with prefix after 3rd-party packages;
     # it's a comma-separated list of prefixes
-    local-prefixes: github.com/onflow/flow-go
+    local-prefixes: github.com/onflow/flow-go/
 
   gosec:
     # To select a subset of rules to run.

CODEOWNERS

@@ -1,52 +1,5 @@
-# Collection Stream
-/cmd/collection/** @jordanschalm
-/engine/collection/** @jordanschalm
-/protobuf/services/collection/** @jordanschalm
+# Protocol owners are not generally differentiated by sub-area for simplicity
+/** @onflow/flow-core-protocol
 
-# Consensus Stream
-/cmd/consensus/** @AlexHentschel @durkmurder @jordanschalm
-/engine/consensus/** @AlexHentschel @durkmurder @jordanschalm
-
-# Execution Stream
-/cmd/execution/** @ramtinms
-/engine/execution/** @ramtinms
-
-# Access Stream
-/access/** @peterargue
-/cmd/access/** @peterargue
-/cmd/observer/** @peterargue
-/engine/access/** @peterargue
-
-# Verification Stream
-/cmd/verification/** @ramtinms @yhassanzadeh13
-/engine/verification/** @ramtinms @yhassanzadeh13
-/module/chunking/** @ramtinms
-/integration/tests/verification @ramtinms @yhassanzadeh13
-
-# Ledger Stream
-/ledger/** @ramtinms @AlexHentschel
-
-# FVM Stream
-/fvm/**  @ramtinms @janezpodhostnik
-
-# Networking Stream
-/network/** @yhassanzadeh13
-
-# Cryptography Stream
-/crypto/** @tarakby
-
-# Bootstrap and transit scripts
-/cmd/bootstrap/** @zhangchiqing
-
-# Dev Tools Stream
-.github/workflows/** @gomisha
-/insecure/** @gomisha @yhassanzadeh13
-/integration/benchnet2/** @gomisha
-/tools/test_monitor/** @gomisha
-
-# Performance Stream
-/integration/benchmark/** @gomisha
-
-# Execution Sync
-/module/executiondatasync/** @peterargue
-/module/state_synchronization/** @peterargue
+# FVM specific
+/fvm/** @janezpodhostnik @zhangchiqing 

Makefile

@@ -26,7 +26,7 @@ IMAGE_TAG_ARM := $(IMAGE_TAG)-arm
 # Name of the cover profile
 COVER_PROFILE := coverage.txt
 # Disable go sum database lookup for private repos
-GOPRIVATE=github.com/dapperlabs/*
+GOPRIVATE=github.com/onflow/*-internal
 # OS
 UNAME := $(shell uname)
 
@@ -101,7 +101,7 @@ go-math-rand-check:
 	#   - "onflow/crypto/random" for deterministic randomness
 	grep --include=\*.go \
 	--exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \
-    --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \
+    --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* --exclude-dir=*emulator* -rnw '"math/rand"'; \
     if [ $$? -ne 1 ]; then \
        echo "[Error] Go production code should not use math/rand package"; exit 1; \
     fi
@@ -138,8 +138,8 @@ endif
 
 .PHONY: generate-openapi
 generate-openapi:
-	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/rest/models;
-	go fmt ./engine/access/rest/models
+	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/rest/http/models;
+	go fmt ./engine/access/rest/http/models
 
 .PHONY: generate
 generate: generate-proto generate-mocks generate-fvm-env-wrappers
@@ -203,10 +203,13 @@ generate-mocks: install-mock-generators
 	mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
 	mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
 	mockery --name 'BlockTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
+	mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'LinkGenerator' --dir="./engine/access/rest/common/models" --case=underscore --output="./engine/access/rest/common/models/mock"  --outpkg="mock"
+	mockery --name 'WebsocketConnection' --dir="./engine/access/rest/websockets" --case=underscore --output="./engine/access/rest/websockets/mock"  --outpkg="mock"
 	mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
 	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
 	mockery --name 'Communicator' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
-
 	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
 	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
 	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
@@ -857,7 +860,8 @@ docker-all-tools: tool-util tool-remove-execution-fork
 
 PHONY: docker-build-util
 docker-build-util:
-	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
 		-t "$(CONTAINER_REGISTRY)/util:latest"  \
 		-t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 

README.md

@@ -54,7 +54,7 @@ The following table lists all work streams and links to their home directory and
 ## Installation
 
 - Clone this repository
-- Install [Go](https://golang.org/doc/install) (Flow supports Go 1.18 and later)
+- Install [Go](https://golang.org/doc/install) (Flow requires Go 1.22 and later)
 - Install [Docker](https://docs.docker.com/get-docker/), which is used for running a local network and integration tests
 - Make sure the [`GOPATH`](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable) and `GOBIN` environment variables
   are set, and `GOBIN` is added to your path:

access/api.go

@@ -203,10 +203,50 @@ type API interface {
 	//
 	// If invalid parameters will be supplied SubscribeBlockDigestsFromLatest will return a failed subscription.
 	SubscribeBlockDigestsFromLatest(ctx context.Context, blockStatus flow.BlockStatus) subscription.Subscription
-	// SubscribeTransactionStatuses streams transaction statuses starting from the reference block saved in the
-	// transaction itself until the block containing the transaction becomes sealed or expired. When the transaction
-	// status becomes TransactionStatusSealed or TransactionStatusExpired, the subscription will automatically shut down.
-	SubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
+	// SubscribeTransactionStatusesFromStartBlockID subscribes to transaction status updates for a given transaction ID.
+	// Monitoring begins from the specified block ID. The subscription streams status updates until the transaction
+	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
+	// these final statuses, the subscription will automatically terminate.
+	//
+	// Parameters:
+	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
+	//   - txID: The identifier of the transaction to monitor.
+	//   - startBlockID: The block ID from which to start monitoring.
+	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
+	SubscribeTransactionStatusesFromStartBlockID(ctx context.Context, txID flow.Identifier, startBlockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
+	// SubscribeTransactionStatusesFromStartHeight subscribes to transaction status updates for a given transaction ID.
+	// Monitoring begins from the specified block height. The subscription streams status updates until the transaction
+	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
+	// these final statuses, the subscription will automatically terminate.
+	//
+	// Parameters:
+	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
+	//   - txID: The unique identifier of the transaction to monitor.
+	//   - startHeight: The block height from which to start monitoring.
+	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
+	SubscribeTransactionStatusesFromStartHeight(ctx context.Context, txID flow.Identifier, startHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
+	// SubscribeTransactionStatusesFromLatest subscribes to transaction status updates for a given transaction ID.
+	// Monitoring begins from the latest block. The subscription streams status updates until the transaction
+	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). When the transaction reaches one of
+	// these final statuses, the subscription will automatically terminate.
+	//
+	// Parameters:
+	//   - ctx: The context to manage the subscription's lifecycle, including cancellation.
+	//   - txID: The unique identifier of the transaction to monitor.
+	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
+	SubscribeTransactionStatusesFromLatest(ctx context.Context, txID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
+	// SendAndSubscribeTransactionStatuses sends a transaction to the network and subscribes to its status updates.
+	// Monitoring begins from the reference block saved in the transaction itself and streams status updates until the transaction
+	// reaches a final state (TransactionStatusSealed or TransactionStatusExpired). Once a final status is reached, the subscription
+	// automatically terminates.
+	//
+	// Parameters:
+	//   - ctx: The context to manage the transaction sending and subscription lifecycle, including cancellation.
+	//   - tx: The transaction body to be sent and monitored.
+	//   - requiredEventEncodingVersion: The version of event encoding required for the subscription.
+	//
+	// If the transaction cannot be sent, the subscription will fail and return a failed subscription.
+	SendAndSubscribeTransactionStatuses(ctx context.Context, tx *flow.TransactionBody, requiredEventEncodingVersion entities.EventEncodingVersion) subscription.Subscription
 }
 
 // TODO: Combine this with flow.TransactionResult?