Merge remote-tracking branch 'upstream/master' into ian/max-message-size

.circleci/config.yml

@@ -79,7 +79,7 @@ slack-fail-stop-step: &slack-fail-post-step
 # ===== Workflow Definitions =====
 workflows:
   version: 2
-  "circleci_build_and_test":
+  nightly_build_and_test:
     jobs:
       - build_nightly:
           name: << matrix.platform >>_build_nightly
@@ -94,19 +94,6 @@ workflows:
           context: slack-secrets
           <<: *slack-fail-post-step
 
-      - test:
-          name: << matrix.platform >>_test
-          matrix: &matrix-default
-            parameters:
-              platform: ["amd64", "arm64"]
-          filters: &filters-default
-            branches:
-              ignore:
-                - /rel\/.*/
-                - << pipeline.parameters.valid_nightly_branch >>
-          context: slack-secrets
-          <<: *slack-fail-post-step
-
       - test_nightly:
           name: << matrix.platform >>_test_nightly
           matrix:
@@ -116,15 +103,6 @@ workflows:
           context: slack-secrets
           <<: *slack-fail-post-step
 
-      - integration:
-          name: << matrix.platform >>_integration
-          matrix:
-            <<: *matrix-default
-          filters:
-            <<: *filters-default
-          context: slack-secrets
-          <<: *slack-fail-post-step
-
       - integration_nightly:
           name: << matrix.platform >>_integration_nightly
           matrix:
@@ -134,15 +112,6 @@ workflows:
           context: slack-secrets
           <<: *slack-fail-post-step
 
-      - e2e_expect:
-          name: << matrix.platform >>_e2e_expect
-          matrix:
-            <<: *matrix-default
-          filters:
-            <<: *filters-default
-          context: slack-secrets
-          <<: *slack-fail-post-step
-
       - e2e_expect_nightly:
           name: << matrix.platform >>_e2e_expect_nightly
           matrix:
@@ -152,15 +121,6 @@ workflows:
           context: slack-secrets
           <<: *slack-fail-post-step
 
-      - e2e_subs:
-          name: << matrix.platform >>_e2e_subs
-          matrix:
-            <<: *matrix-default
-          filters:
-            <<: *filters-default
-          context: slack-secrets
-          <<: *slack-fail-post-step
-
       - e2e_subs_nightly:
           name: << matrix.platform >>_e2e_subs_nightly
           matrix:
@@ -172,17 +132,6 @@ workflows:
             - aws-secrets
           <<: *slack-fail-post-step
 
-      - tests_verification_job:
-          name: << matrix.platform >>_<< matrix.job_type >>_verification
-          matrix:
-            parameters:
-              platform: ["amd64", "arm64"]
-              job_type: ["test", "integration", "e2e_expect"]
-          requires:
-            - << matrix.platform >>_<< matrix.job_type >>
-          context: slack-secrets
-          <<: *slack-fail-post-step
-
       - tests_verification_job_nightly:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
           matrix:
@@ -203,17 +152,63 @@ workflows:
             - << matrix.platform >>_integration_nightly_verification
             - << matrix.platform >>_e2e_expect_nightly_verification
             - << matrix.platform >>_e2e_subs_nightly
-          filters:
-            branches:
-              only:
-                - /rel\/.*/
-                - << pipeline.parameters.valid_nightly_branch >>
           context:
             - slack-secrets
             - aws-secrets
           <<: *slack-fail-post-step
 
-      #- windows_x64_build
+  "circleci_build_and_test":
+    jobs:
+      - test:
+          name: << matrix.platform >>_test
+          matrix: &matrix-default
+            parameters:
+              platform: ["amd64"]
+          filters: &filters-default
+            branches:
+              ignore:
+                - /rel\/.*/
+                - << pipeline.parameters.valid_nightly_branch >>
+          context: slack-secrets
+          <<: *slack-fail-post-step
+
+      - integration:
+          name: << matrix.platform >>_integration
+          matrix:
+            <<: *matrix-default
+          filters:
+            <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
+
+      - e2e_expect:
+          name: << matrix.platform >>_e2e_expect
+          matrix:
+            <<: *matrix-default
+          filters:
+            <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
+
+      - e2e_subs:
+          name: << matrix.platform >>_e2e_subs
+          matrix:
+            <<: *matrix-default
+          filters:
+            <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
+
+      - tests_verification_job:
+          name: << matrix.platform >>_<< matrix.job_type >>_verification
+          matrix:
+            parameters:
+              platform: ["amd64"]
+              job_type: ["test", "integration", "e2e_expect"]
+          requires:
+            - << matrix.platform >>_<< matrix.job_type >>
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
 # ===== Job Definitions =====
 jobs:
@@ -606,6 +601,7 @@ commands:
             export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
             export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
             export PARTITION_ID=${CIRCLE_NODE_INDEX}
+            export GOEXPERIMENT="none"
             gotestsum --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
       - store_artifacts:
           path: << parameters.result_path >>

.github/workflows/benchmarks.yml

@@ -29,7 +29,7 @@ jobs:
       # degradation, (2) `eval` degradation.  (2) suggests a broader performance
       # issue. 
       - name: Run benchmark
-        run: go test ./data/transactions/logic -bench 'BenchmarkUintMath' | tee benchmark_output.txt
+        run: go test ./data/transactions/logic -bench 'BenchmarkUintMath' -run=^$ | tee benchmark_output.txt
       - name: Push benchmark result to gh-pages branch
         if: github.event_name == 'push'
         uses: benchmark-action/github-action-benchmark@v1

.golangci.yml

@@ -33,6 +33,8 @@ linters-settings:
     exclude-functions:
     # We do this 121 times and never check the error.
     - (*github.com/spf13/cobra.Command).MarkFlagRequired
+    - (*github.com/spf13/pflag.FlagSet).MarkDeprecated
+    - (*github.com/spf13/pflag.FlagSet).MarkShorthandDeprecated
   govet:
     check-shadowing: true
     settings:

Makefile

@@ -77,6 +77,11 @@ ifeq ($(SHORT_PART_PERIOD), 1)
 export SHORT_PART_PERIOD_FLAG := -s
 endif
 
+# Disable go experiments during build as of go 1.20.5 due to
+# https://github.com/golang/go/issues/60825
+# Likely fix: https://go-review.googlesource.com/c/go/+/503937/6/src/runtime/race_arm64.s
+export GOEXPERIMENT=none
+
 GOTAGS      := --tags "$(GOTAGSLIST)"
 GOTRIMPATH	:= $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath)
 
@@ -158,7 +163,7 @@ ALWAYS:
 # build our fork of libsodium, placing artifacts into crypto/lib/ and crypto/include/
 crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 	mkdir -p crypto/copies/$(OS_TYPE)/$(ARCH)
-	cp -R crypto/libsodium-fork crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork
+	cp -R crypto/libsodium-fork/. crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork
 	cd crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork && \
 		./autogen.sh --prefix $(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH) && \
 		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" && \

agreement/agreementtest/simulate_test.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/algorand/go-deadlock"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
 
 	"github.com/algorand/go-algorand/agreement"
 	"github.com/algorand/go-algorand/config"
@@ -124,24 +125,10 @@ func makeTestLedger(state map[basics.Address]basics.AccountData) agreement.Ledge
 func (l *testLedger) copy() *testLedger {
 	dup := new(testLedger)
 
-	dup.entries = make(map[basics.Round]bookkeeping.Block)
-	dup.certs = make(map[basics.Round]agreement.Certificate)
-	dup.state = make(map[basics.Address]basics.AccountData)
-	dup.notifications = make(map[basics.Round]signal)
-
-	for k, v := range l.entries {
-		dup.entries[k] = v
-	}
-	for k, v := range l.certs {
-		dup.certs[k] = v
-	}
-	for k, v := range l.state {
-		dup.state[k] = v
-	}
-	for k, v := range dup.notifications {
-		// note that old opened channels will now fire when these are closed
-		dup.notifications[k] = v
-	}
+	dup.entries = maps.Clone(l.entries)
+	dup.certs = maps.Clone(l.certs)
+	dup.state = maps.Clone(l.state)
+	dup.notifications = maps.Clone(l.notifications) // old opened channels will now fire when these are closed
 	dup.nextRound = l.nextRound
 
 	return dup

agreement/autopsy.go

@@ -24,6 +24,7 @@ import (
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
+	"golang.org/x/exp/slices"
 )
 
 // An Autopsy is a trace of the ordered input events and output
@@ -102,9 +103,7 @@ func (m *multiCloser) Close() error {
 
 // makeMultiCloser returns a Closer that closes all the given closers.
 func makeMultiCloser(closers ...io.Closer) io.Closer {
-	r := make([]io.Closer, len(closers))
-	copy(r, closers)
-	return &multiCloser{r}
+	return &multiCloser{slices.Clone(closers)}
 }
 
 type autopsyTrace struct {

agreement/common_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/algorand/go-deadlock"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
@@ -207,10 +208,7 @@ func makeTestLedger(state map[basics.Address]basics.AccountData) Ledger {
 	l.certs = make(map[basics.Round]Certificate)
 	l.nextRound = 1
 
-	l.state = make(map[basics.Address]basics.AccountData)
-	for k, v := range state {
-		l.state[k] = v
-	}
+	l.state = maps.Clone(state)
 
 	l.notifications = make(map[basics.Round]signal)
 
@@ -226,10 +224,7 @@ func makeTestLedgerWithConsensusVersion(state map[basics.Address]basics.AccountD
 	l.certs = make(map[basics.Round]Certificate)
 	l.nextRound = 1
 
-	l.state = make(map[basics.Address]basics.AccountData)
-	for k, v := range state {
-		l.state[k] = v
-	}
+	l.state = maps.Clone(state)
 
 	l.notifications = make(map[basics.Round]signal)
 
@@ -245,10 +240,7 @@ func makeTestLedgerMaxBlocks(state map[basics.Address]basics.AccountData, maxNum
 
 	l.maxNumBlocks = maxNumBlocks
 
-	l.state = make(map[basics.Address]basics.AccountData)
-	for k, v := range state {
-		l.state[k] = v
-	}
+	l.state = maps.Clone(state)
 
 	l.notifications = make(map[basics.Round]signal)
 

cmd/algokey/keyreg.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/maps"
 
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/account"
@@ -94,10 +95,7 @@ func init() {
 		"betanet": mustConvertB64ToDigest("mFgazF+2uRS1tMiL9dsj01hJGySEmPN28B/TjjvpVW0="),
 		"devnet":  mustConvertB64ToDigest("sC3P7e2SdbqKJK0tbiCdK9tdSpbe6XeCGKdoNzmlj0E="),
 	}
-	validNetworkList = make([]string, 0, len(validNetworks))
-	for k := range validNetworks {
-		validNetworkList = append(validNetworkList, k)
-	}
+	validNetworkList = maps.Keys(validNetworks)
 }
 
 func mustConvertB64ToDigest(b64 string) (digest crypto.Digest) {

cmd/goal/account.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 
 	"github.com/algorand/go-algorand/cmd/util/datadir"
 	"github.com/algorand/go-algorand/config"
@@ -557,35 +558,31 @@ var infoCmd = &cobra.Command{
 func printAccountInfo(client libgoal.Client, address string, onlyShowAssetIds bool, account model.Account) bool {
 	var createdAssets []model.Asset
 	if account.CreatedAssets != nil {
-		createdAssets = make([]model.Asset, len(*account.CreatedAssets))
-		copy(createdAssets, *account.CreatedAssets)
+		createdAssets = slices.Clone(*account.CreatedAssets)
 		sort.Slice(createdAssets, func(i, j int) bool {
 			return createdAssets[i].Index < createdAssets[j].Index
 		})
 	}
 
 	var heldAssets []model.AssetHolding
 	if account.Assets != nil {
-		heldAssets = make([]model.AssetHolding, len(*account.Assets))
-		copy(heldAssets, *account.Assets)
+		heldAssets = slices.Clone(*account.Assets)
 		sort.Slice(heldAssets, func(i, j int) bool {
 			return heldAssets[i].AssetID < heldAssets[j].AssetID
 		})
 	}
 
 	var createdApps []model.Application
 	if account.CreatedApps != nil {
-		createdApps = make([]model.Application, len(*account.CreatedApps))
-		copy(createdApps, *account.CreatedApps)
+		createdApps = slices.Clone(*account.CreatedApps)
 		sort.Slice(createdApps, func(i, j int) bool {
 			return createdApps[i].Id < createdApps[j].Id
 		})
 	}
 
 	var optedInApps []model.ApplicationLocalState
 	if account.AppsLocalState != nil {
-		optedInApps = make([]model.ApplicationLocalState, len(*account.AppsLocalState))
-		copy(optedInApps, *account.AppsLocalState)
+		optedInApps = slices.Clone(*account.AppsLocalState)
 		sort.Slice(optedInApps, func(i, j int) bool {
 			return optedInApps[i].Id < optedInApps[j].Id
 		})