tools: block-generator applications. Part 1: create (#5450)

.gitignore

@@ -74,3 +74,6 @@ tools/block-generator/block-generator
 
 # cross repo types tool binary
 tools/x-repo-types/x-repo-types
+
+# python virtual environment
+.venv

tools/block-generator/Makefile

@@ -0,0 +1,36 @@
+SCENARIO = scenarios/config.app.create.yml # test_config.yml
+SKIP = --skip-runner
+RESETDB = --reset-db
+REPORTS = ../../tmp/RUN_RUNNER_OUTPUTS
+DURATION = 30s
+
+debug-blockgen:
+	python run_runner.py \
+	--conduit-binary ./conduit \
+	--scenario $(SCENARIO) \
+	--report-directory $(REPORTS) \
+	--keep-alive $(SKIP) \
+	--test-duration $(DURATION) \
+	$(RESETDB)
+
+clean-reports:
+	rm -rf $(REPORTS) 
+
+cleanup: clean-reports
+	python run_runner.py --purge
+
+enter-pg:
+	docker exec -it generator-test-container psql -U algorand -d generator_db
+
+run-runner:
+	./block-generator runner --conduit-binary ./conduit \
+	--log-level trace \
+	--keep-data-dir \
+	--test-duration $(DURATION) \
+	--log-level trace \
+	--postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \
+	--scenario $(SCENARIO) \
+	$(RESETDB) \
+	--report-directory $(REPORTS)
+
+

tools/block-generator/generator/generate.go

@@ -17,6 +17,7 @@
 package generator
 
 import (
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -40,6 +41,20 @@ import (
 	"github.com/algorand/go-algorand/data/transactions"
 )
 
+// ---- templates ----
+
+//go:embed teal/poap_boxes.teal
+var approvalBoxes string
+
+//go:embed teal/poap_clear.teal
+var clearBoxes string
+
+//go:embed teal/swap_amm.teal
+var approvalSwap string
+
+//go:embed teal/swap_clear.teal
+var clearSwap string
+
 // ---- constructors ----
 
 // MakeGenerator initializes the Generator object.
@@ -78,6 +93,9 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat
 		gen.genesisHash = bkGenesis.Hash()
 	}
 
+	gen.apps = make(map[appKind][]*appData)
+	gen.pendingApps = make(map[appKind][]*appData)
+
 	gen.initializeAccounting()
 	gen.initializeLedger()
 	for _, val := range getTransactionOptions() {
@@ -166,10 +184,6 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat
 
 // initializeAccounting creates the genesis accounts.
 func (g *generator) initializeAccounting() {
-	if g.config.NumGenesisAccounts == 0 {
-		panic("Number of genesis accounts must be > 0.")
-	}
-
 	g.numPayments = 0
 	g.numAccounts = g.config.NumGenesisAccounts
 	for i := uint64(0); i < g.config.NumGenesisAccounts; i++ {
@@ -483,18 +497,6 @@ func (g *generator) Stop() {
 	g.ledger.Close()
 }
 
-// Accounts is used in the runner to generate a list of addresses.
-func (g *generator) Accounts() <-chan basics.Address {
-	results := make(chan basics.Address, 10)
-	go func() {
-		defer close(results)
-		for i := uint64(0); i < g.numAccounts; i++ {
-			results <- indexToAccount(i)
-		}
-	}()
-	return results
-}
-
 // ---- transaction options vectors ----
 
 func getTransactionOptions() []interface{} {
@@ -529,6 +531,8 @@ func (g *generator) generateTransaction(round uint64, intra uint64) (transaction
 		return g.generatePaymentTxn(round, intra)
 	case assetTx:
 		return g.generateAssetTxn(round, intra)
+	case applicationTx:
+		return g.generateAppTxn(round, intra)
 	default:
 		return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("no generator available for %s", selection)
 	}
@@ -539,7 +543,7 @@ func (g *generator) generateTransaction(round uint64, intra uint64) (transaction
 // generatePaymentTxn creates a new payment transaction. The sender is always a genesis account, the receiver is random,
 // or a new account.
 func (g *generator) generatePaymentTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) {
-	selection, err := weightedSelection(g.payTxWeights, getPaymentTxOptions(), paymentTx)
+	selection, err := weightedSelection(g.payTxWeights, getPaymentTxOptions(), paymentPayTx)
 	if err != nil {
 		return transactions.SignedTxn{}, transactions.ApplyData{}, err
 	}
@@ -556,7 +560,7 @@ func (g *generator) generatePaymentTxnInternal(selection TxTypeID, round uint64,
 	// Select a receiver
 	var receiveIndex uint64
 	switch selection {
-	case paymentTx:
+	case paymentPayTx:
 		receiveIndex = rand.Uint64() % g.numAccounts
 	case paymentAcctCreateTx:
 		// give new accounts get extra algos for sending other transactions
@@ -598,6 +602,7 @@ func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.S
 	actual, txn := g.generateAssetTxnInternal(selection.(TxTypeID), round, intra)
 	defer g.recordData(actual, start)
 
+	// TODO: shouldn't we just return an error?
 	if txn.Type == "" {
 		fmt.Println("Empty asset transaction.")
 		os.Exit(1)
@@ -613,14 +618,14 @@ func (g *generator) generateAssetTxnInternal(txType TxTypeID, round uint64, intr
 func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, intra uint64, hintIndex uint64, hint *assetData) (actual TxTypeID, txn transactions.Transaction) {
 	actual = txType
 	// If there are no assets the next operation needs to be a create.
-	if len(g.assets) == 0 {
+	numAssets := uint64(len(g.assets))
+
+	if numAssets == 0 {
 		actual = assetCreate
 	}
-
-	numAssets := uint64(len(g.assets))
 	var senderIndex uint64
 	if actual == assetCreate {
-		numAssets = uint64(len(g.assets)) + uint64(len(g.pendingAssets))
+		numAssets += uint64(len(g.pendingAssets))
 		senderIndex = numAssets % g.config.NumGenesisAccounts
 		senderAcct := indexToAccount(senderIndex)
 
@@ -643,11 +648,14 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64,
 
 		g.pendingAssets = append(g.pendingAssets, &a)
 	} else {
-		assetIndex := rand.Uint64() % numAssets
-		asset := g.assets[assetIndex]
+		var assetIndex uint64
+		var asset *assetData
 		if hint != nil {
 			assetIndex = hintIndex
 			asset = hint
+		} else {
+			assetIndex = rand.Uint64()%numAssets
+			asset = g.assets[assetIndex]
 		}
 
 		switch actual {
@@ -759,6 +767,93 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64,
 	return
 }
 
+// ---- 3. App Transactions ----
+
+func (g *generator) generateAppTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) {
+	start := time.Now()
+	selection, err := weightedSelection(g.appTxWeights, getAppTxOptions(), appSwapCall)
+	if err != nil {
+		return transactions.SignedTxn{}, transactions.ApplyData{}, err
+	}
+
+	actual, txn, err := g.generateAppCallInternal(selection.(TxTypeID), round, intra, 0, nil)
+	if err != nil {
+		return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("unexpected error received from generateAppCallInternal(): %w", err)
+	}
+	if txn.Type == "" {
+		return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("missing transaction type for app transaction")
+	}
+
+	g.recordData(actual, start)
+	return signTxn(txn), transactions.ApplyData{}, nil
+}
+
+func (g *generator) generateAppCallInternal(txType TxTypeID, round, intra, hintIndex uint64, hintApp *appData) (TxTypeID, transactions.Transaction, error) {
+	actual := txType
+
+	isApp, kind, appTx, err := parseAppTxType(txType)
+	if err != nil {
+		return "", transactions.Transaction{}, err
+	}
+	if !isApp {
+		return "", transactions.Transaction{}, fmt.Errorf("should be an app but not parsed that way: %v", txType)
+	}
+	if appTx != appTxTypeCreate {
+		return "", transactions.Transaction{}, fmt.Errorf("invalid transaction type for app %v", appTx)
+	}
+
+	var senderIndex uint64
+	if hintApp != nil {
+		return "", transactions.Transaction{}, fmt.Errorf("not ready for hint app %v", hintApp)
+	} else {
+		senderIndex = rand.Uint64() % g.numAccounts
+	}
+
+	actualAppTx := appTx
+
+	numApps := uint64(len(g.apps[kind]))
+	if numApps == 0 {
+		actualAppTx = appTxTypeCreate
+	}
+
+	var txn transactions.Transaction
+	if actualAppTx == appTxTypeCreate {
+		numApps += uint64(len(g.pendingApps[kind]))
+		senderIndex = numApps % g.config.NumGenesisAccounts
+		senderAcct := indexToAccount(senderIndex)
+
+		var approval, clear string
+		if kind == appKindSwap {
+			approval, clear = approvalSwap, clearSwap
+		} else {
+			approval, clear = approvalBoxes, clearBoxes
+		}
+
+		txn = g.makeAppCreateTxn(senderAcct, round, intra, approval, clear)
+
+		appID := g.txnCounter + intra + 1
+		holding := &appHolding{appIndex: appID}
+		ad := &appData{
+			appID:    appID,
+			creator:  senderIndex,
+			kind:     kind,
+			holdings: []*appHolding{holding},
+			holders:  map[uint64]*appHolding{senderIndex: holding},
+		}
+		g.pendingApps[kind] = append(g.pendingApps[kind], ad)
+	}
+
+	// account := indexToAccount(senderIndex)
+	// txn = g.makeAppCallTxn(account, round, intra, round, approval, clear)
+
+	if g.balances[senderIndex] < g.params.MinTxnFee {
+		return "", transactions.Transaction{}, fmt.Errorf("the sender account does not have enough algos for the app call. idx %d, app transaction type %v, num %d\n\n", senderIndex, txType, g.reportData[txType].GenerationCount)
+	}
+	g.balances[senderIndex] -= g.params.MinTxnFee
+
+	return actual, txn, nil
+}
+
 // ---- miscellaneous ----
 
 func track(id TxTypeID) (TxTypeID, time.Time) {
@@ -790,6 +885,12 @@ func (g *generator) finishRound(txnCount uint64) {
 	// Apply pending assets...
 	g.assets = append(g.assets, g.pendingAssets...)
 	g.pendingAssets = nil
+
+	// Apply pending apps...
+	for _, kind := range []appKind{appKindSwap, appKindBoxes} {
+		g.apps[kind] = append(g.apps[kind], g.pendingApps[kind]...)
+		g.pendingApps[kind] = nil
+	}
 }
 
 func signTxn(txn transactions.Transaction) transactions.SignedTxn {

tools/block-generator/generator/generate_test.go

@@ -191,6 +191,49 @@ func TestAssetDestroy(t *testing.T) {
 	require.Len(t, g.assets, 0)
 }
 
+func TestAppCreate(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+
+	g := makePrivateGenerator(t, 0, bookkeeping.Genesis{})
+
+	// app call transaction creating appBoxes
+	actual, txn, err := g.generateAppCallInternal(appBoxesCreate, 1, 0, 0, nil)
+	require.NoError(t, err)
+	require.Equal(t, appBoxesCreate, actual)
+	require.Equal(t, protocol.ApplicationCallTx, txn.Type)
+	require.Len(t, g.apps, 0)
+	require.Len(t, g.pendingApps, 1)
+	require.Len(t, g.pendingApps[appKindBoxes], 1)
+	require.Len(t, g.pendingApps[appKindSwap], 0)
+	require.Len(t, g.pendingApps[appKindBoxes][0].holdings, 1)
+	require.Len(t, g.pendingApps[appKindBoxes][0].holders, 1)
+	ad := *g.pendingApps[appKindBoxes][0]
+	holding := *ad.holdings[0]
+	require.Equal(t, holding, *ad.holders[0])
+	require.Equal(t, uint64(1001), holding.appIndex)
+	require.Equal(t, ad.appID, holding.appIndex)
+	require.Equal(t, appKindBoxes, ad.kind)
+
+	// app call transaction creating appSwap
+	actual, txn, err = g.generateAppCallInternal(appSwapCreate, 1, 0, 0, nil)
+	require.NoError(t, err)
+	require.Equal(t, appSwapCreate, actual)
+	require.Equal(t, protocol.ApplicationCallTx, txn.Type)
+	require.Len(t, g.apps, 0)
+	require.Len(t, g.pendingApps, 2)
+	require.Len(t, g.pendingApps[appKindBoxes], 1)
+	require.Len(t, g.pendingApps[appKindSwap], 1)
+	require.Len(t, g.pendingApps[appKindSwap][0].holdings, 1)
+	require.Len(t, g.pendingApps[appKindSwap][0].holders, 1)
+	ad = *g.pendingApps[appKindSwap][0]
+	holding = *ad.holdings[0]
+	require.Equal(t, holding, *ad.holders[0])
+	require.Equal(t, uint64(1001), holding.appIndex)
+	require.Equal(t, ad.appID, holding.appIndex)
+	require.Equal(t, appKindSwap, ad.kind)
+}
+
 func TestWriteRoundZero(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	var testcases = []struct {
@@ -214,7 +257,7 @@ func TestWriteRoundZero(t *testing.T) {
 	}
 	for _, tc := range testcases {
 		tc := tc
-		t.Run(fmt.Sprintf("%s", tc.name), func(t *testing.T) {
+		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 			g := makePrivateGenerator(t, tc.dbround, tc.genesis)
 			var data []byte
@@ -408,7 +451,9 @@ func TestHandlers(t *testing.T) {
 	}
 
 	for _, testcase := range testcases {
+		testcase := testcase
 		t.Run(testcase.name, func(t *testing.T) {
+			t.Parallel()
 			req := httptest.NewRequest("GET", testcase.url, nil)
 			w := httptest.NewRecorder()
 			handler(w, req)

tools/block-generator/generator/generator_types.go

@@ -34,9 +34,8 @@ type Generator interface {
 	WriteGenesis(output io.Writer) error
 	WriteBlock(output io.Writer, round uint64) error
 	WriteAccount(output io.Writer, accountString string) error
-	WriteStatus(output io.Writer) error
 	WriteDeltas(output io.Writer, round uint64) error
-	Accounts() <-chan basics.Address
+	WriteStatus(output io.Writer) error
 	Stop()
 }
 
@@ -115,6 +114,11 @@ type assetData struct {
 type appData struct {
 	appID   uint64
 	creator uint64
+	kind    appKind
+	// Holding at index 0 is the creator.
+	holdings []*appHolding
+	// Set of holders in the holdings array for easy reference.
+	holders map[uint64]*appHolding
 	// TODO: more data, not sure yet exactly what
 }
 
@@ -123,6 +127,11 @@ type assetHolding struct {
 	balance   uint64
 }
 
+type appHolding struct {
+	appIndex uint64
+	// TODO: more data, not sure yet exactly what
+}
+
 // Report is the generation report.
 type Report map[TxTypeID]TxData