diff --git a/tools/block-generator/Makefile b/tools/block-generator/Makefile index 7c4f0c0946..ac2f997244 100644 --- a/tools/block-generator/Makefile +++ b/tools/block-generator/Makefile @@ -1,8 +1,15 @@ -SCENARIO = scenarios/config.app.create.yml # test_config.yml +SCENARIO = scenarios/config.allmixed.small.yml SKIP = --skip-runner RESETDB = --reset-db REPORTS = ../../tmp/RUN_RUNNER_OUTPUTS DURATION = 30s +VERBOSE = # --verbose + +block-generator: clean-generator + go build + +clean-generator: + rm -f block-generator debug-blockgen: python run_runner.py \ @@ -13,24 +20,32 @@ debug-blockgen: --test-duration $(DURATION) \ $(RESETDB) -clean-reports: - rm -rf $(REPORTS) - -cleanup: clean-reports - python run_runner.py --purge - enter-pg: docker exec -it generator-test-container psql -U algorand -d generator_db -run-runner: +clean-docker: + docker rm -f generator-test-container + +run-runner: block-generator ./block-generator runner --conduit-binary ./conduit \ - --log-level trace \ --keep-data-dir \ --test-duration $(DURATION) \ - --log-level trace \ + --conduit-log-level trace \ --postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \ --scenario $(SCENARIO) \ $(RESETDB) \ + $(VERBOSE) \ --report-directory $(REPORTS) +clean-reports: + rm -rf $(REPORTS) + +pre-git-push: + mv _go.mod go.mod + mv _go.sum go.sum + cd ../../ && make tidy +post-git-push: + mv go.mod _go.mod + mv go.sum _go.sum + cd ../../ && make tidy && go get github.com/lib/pq diff --git a/tools/block-generator/README.md b/tools/block-generator/README.md index 7e38d3d78b..d4d8d9187b 100644 --- a/tools/block-generator/README.md +++ b/tools/block-generator/README.md @@ -133,18 +133,21 @@ Usage: Flags: -i, --conduit-binary string Path to conduit binary. - --cpuprofile string Path where conduit writes its CPU profile. + -l, --conduit-log-level string LogLevel to use when starting Conduit. [panic, fatal, error, warn, info, debug, trace] (default "error") + --cpuprofile string Path where Conduit writes its CPU profile. + -f, --genesis-file string file path to the genesis associated with the db snapshot -h, --help help for runner -k, --keep-data-dir If set the validator will not delete the data directory after tests complete. - -l, --log-level string LogLevel to use when starting conduit. [panic, fatal, error, warn, info, debug, trace] (default "error") -p, --metrics-port uint Port to start the metrics server at. (default 9999) -c, --postgres-connection-string string Postgres connection string. -r, --report-directory string Location to place test reports. - --reset If set any existing report directory will be deleted before running tests. + --reset-db If set database will be deleted before running tests. + --reset-report-dir If set any existing report directory will be deleted before running tests. -s, --scenario string Directory containing scenarios, or specific scenario file. -d, --test-duration duration Duration to use for each scenario. (default 5m0s) --validate If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure. -``` + -v, --verbose If set the runner will print debugging information from the generator and ledger. + ``` ## Example Run using Conduit and Postgres in **bash** via `run_runner.sh` @@ -180,5 +183,5 @@ Then you can execute the following command to run the scenario: ### Scenario Report -If all goes well, the run will generate a directory `tmp/OUTPUT_RUN_RUNNER_TEST` +If all goes well, the run will generate a directory `../../tmp/OUTPUT_RUN_RUNNER_TEST` and in that directory you can see the statistics of the run in `scenario.report`. diff --git a/tools/block-generator/generator/config.go b/tools/block-generator/generator/config.go index 97f45f2b34..c8f4ffd23e 100644 --- a/tools/block-generator/generator/config.go +++ b/tools/block-generator/generator/config.go @@ -70,15 +70,19 @@ const ( appBoxesClose TxTypeID = "app_boxes_close" appBoxesClear TxTypeID = "app_boxes_clear" - // TODO: consider an app that creates/destroys an app during opup + // Special TxTypeID's recording effects of higher level transactions + effectPaymentTxSibling TxTypeID = "effect_payment_sibling" + effectInnerTx TxTypeID = "effect_inner_tx" // Defaults defaultGenesisAccountsCount uint64 = 1000 - defaultGenesisAccountInitialBalance uint64 = 1000000000000 + defaultGenesisAccountInitialBalance uint64 = 1_000_000_000000 // 1 million algos per account - assetTotal uint64 = 100000000000000000 + assetTotal uint64 = 100_000_000_000_000_000 // 100 billion units per asset - consensusTimeMilli int64 = 3300 + consensusTimeMilli int64 = 3300 + + // TODO: do we still need this as can get it from the Ledger? startingTxnCounter uint64 = 1000 ) @@ -89,6 +93,18 @@ const ( appKindBoxes ) +func (a appKind) String() string { + switch a { + case appKindSwap: + return "swap" + case appKindBoxes: + return "boxes" + default: + // Return a default value for unknown kinds. + return "Unknown" + } +} + type appTxType uint8 const ( @@ -101,6 +117,28 @@ const ( appTxTypeClear ) +func (a appTxType) String() string { + switch a { + case appTxTypeCreate: + return "create" + case appTxTypeUpdate: + return "update" + case appTxTypeDelete: + return "delete" + case appTxTypeOptin: + return "optin" + case appTxTypeCall: + return "call" + case appTxTypeClose: + return "close" + case appTxTypeClear: + return "clear" + default: + // Return a default value for unknown types. + return "Unknown" + } +} + func parseAppTxType(txType TxTypeID) (isApp bool, kind appKind, tx appTxType, err error) { parts := strings.Split(string(txType), "_") @@ -149,6 +187,10 @@ func parseAppTxType(txType TxTypeID) (isApp bool, kind appKind, tx appTxType, er return } +func getAppTxType(kind appKind, appType appTxType) TxTypeID { + return TxTypeID(fmt.Sprintf("app_%s_%s", kind, appType)) +} + // GenerationConfig defines the tunable parameters for block generation. type GenerationConfig struct { Name string `yaml:"name"` diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 20d21626a8..3475b6250f 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -18,6 +18,7 @@ package generator import ( _ "embed" + "encoding/binary" "encoding/json" "fmt" "io" @@ -25,6 +26,7 @@ import ( "os" "time" + "github.com/algorand/avm-abi/apps" "github.com/algorand/go-algorand/agreement" cconfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/ledger" @@ -38,7 +40,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" - "github.com/algorand/go-algorand/data/transactions" + txn "github.com/algorand/go-algorand/data/transactions" ) // ---- templates ---- @@ -55,16 +57,40 @@ var approvalSwap string //go:embed teal/swap_clear.teal var clearSwap string +// ---- init ---- + +// effects is a map that contains the hard-coded non-trivial +// consequents of a transaction type. +// The "sibling" transactions are added to an atomic transaction group +// in a "makeXyzTransaction" function defined in make_transactions.go. +// The "inner" transactions are created inside the TEAL programs. See: +// * teal/poap_boxes.teal +// * teal/swap_amm.teal +// +// appBoxesCreate: 1 sibling payment tx +// appBoxesOptin: 1 sibling payment tx, 2 inner tx +var effects = map[TxTypeID][]TxEffect{ + appBoxesCreate: { + {effectPaymentTxSibling, 1}, + }, + appBoxesOptin: { + {effectPaymentTxSibling, 1}, + {effectInnerTx, 2}, + }, +} + + // ---- constructors ---- // MakeGenerator initializes the Generator object. -func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config GenerationConfig) (Generator, error) { +func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config GenerationConfig, verbose bool) (Generator, error) { if err := config.validateWithDefaults(false); err != nil { return nil, fmt.Errorf("invalid generator configuration: %w", err) } var proto protocol.ConsensusVersion = "future" gen := &generator{ + verbose: verbose, config: config, protocol: proto, params: cconfig.Consensus[proto], @@ -80,6 +106,7 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat rewardsRate: 0, rewardsRecalculationRound: 0, reportData: make(map[TxTypeID]TxData), + latestData: make(map[TxTypeID]uint64), roundOffset: dbround, } @@ -93,8 +120,19 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat gen.genesisHash = bkGenesis.Hash() } - gen.apps = make(map[appKind][]*appData) - gen.pendingApps = make(map[appKind][]*appData) + gen.resetPendingApps() + gen.appSlice = map[appKind][]*appData{ + appKindBoxes: make([]*appData, 0), + appKindSwap: make([]*appData, 0), + } + gen.appMap = map[appKind]map[uint64]*appData{ + appKindBoxes: make(map[uint64]*appData), + appKindSwap: make(map[uint64]*appData), + } + gen.accountAppOptins = map[appKind]map[uint64][]uint64{ + appKindBoxes: make(map[uint64][]uint64), + appKindSwap: make(map[uint64][]uint64), + } gen.initializeAccounting() gen.initializeLedger() @@ -182,6 +220,17 @@ func MakeGenerator(dbround uint64, bkGenesis bookkeeping.Genesis, config Generat return gen, nil } +func (g *generator) resetPendingApps() { + g.pendingAppSlice = map[appKind][]*appData{ + appKindBoxes: make([]*appData, 0), + appKindSwap: make([]*appData, 0), + } + g.pendingAppMap = map[appKind]map[uint64]*appData{ + appKindBoxes: make(map[uint64]*appData), + appKindSwap: make(map[uint64]*appData), + } +} + // initializeAccounting creates the genesis accounts. func (g *generator) initializeAccounting() { g.numPayments = 0 @@ -283,9 +332,9 @@ func (g *generator) WriteGenesis(output io.Writer) error { // - requested round < offset ---> error // - requested round == offset: the generator will provide a genesis block or offset block // - requested round == generator's round + offset ---> generate a block, -// advance the round, and cache the block in case of repeated requests. +// advance the round, and cache the block in case of repeated requests. // - requested round == generator's round + offset - 1 ---> write the cached block -// but do not advance the round. +// but do not advance the round. // - requested round < generator's round + offset - 1 ---> error // // NOTE: nextRound represents the generator's expectations about the next database round. @@ -320,9 +369,13 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { return nil } // round == nextRound case - + err := g.startRound() + if err != nil { + return err + } numTxnForBlock := g.txnForRound(g.round) + var intra uint64 = 0 var cert rpcs.EncodedBlockCert if g.round == 0 { // we'll write genesis block / offset round for non-empty database @@ -349,46 +402,68 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { CurrentProtocol: g.protocol, }, UpgradeVote: bookkeeping.UpgradeVote{}, - TxnCounter: g.txnCounter + numTxnForBlock, StateProofTracking: nil, } // Generate the transactions - transactions := make([]transactions.SignedTxnInBlock, 0, numTxnForBlock) - for i := uint64(0); i < numTxnForBlock; i++ { - txn, ad, err := g.generateTransaction(g.round, i) + transactions := []txn.SignedTxnInBlock{} + for intra < numTxnForBlock { + var signedTxns []txn.SignedTxn + var ads []txn.ApplyData + var err error + signedTxns, ads, intra, err = g.generateSignedTxns(g.round, intra) if err != nil { - panic(fmt.Sprintf("failed to generate transaction: %v\n", err)) + // return err + return fmt.Errorf("failed to generate transaction: %w", err) } - stib, err := cert.Block.BlockHeader.EncodeSignedTxn(txn, ad) - if err != nil { - panic(fmt.Sprintf("failed to encode transaction: %v\n", err)) + if len(signedTxns) == 0 { + return fmt.Errorf("failed to generate transaction: no transactions given") + } + if len(signedTxns) != len(ads) { + return fmt.Errorf("failed to generate transaction: mismatched number of signed transactions (%d) and apply data (%d)", len(signedTxns), len(ads)) + } + for i, stx := range signedTxns { + stib, err := cert.Block.BlockHeader.EncodeSignedTxn(stx, ads[i]) + if err != nil { + return fmt.Errorf("failed to encode transaction: %w", err) + } + transactions = append(transactions, stib) } - transactions = append(transactions, stib) } - if numTxnForBlock != uint64(len(transactions)) { - panic("Unexpected number of transactions.") + if intra < numTxnForBlock { + return fmt.Errorf("not enough transactions generated: %d > %d", numTxnForBlock, intra) } + cert.Block.BlockHeader.TxnCounter = g.txnCounter + intra cert.Block.Payset = transactions cert.Certificate = agreement.Certificate{} // empty certificate for clarity + var errs []error err := g.ledger.AddBlock(cert.Block, cert.Certificate) if err != nil { - return err + errs = append(errs, fmt.Errorf("error in AddBlock: %w", err)) + } + if g.verbose { + errs2 := g.introspectLedgerVsGenerator(g.round, intra) + if errs2 != nil { + errs = append(errs, errs2...) + } + } + if len(errs) > 0 { + return fmt.Errorf("%d error(s): %v", len(errs), errs) } } cert.Block.BlockHeader.Round = basics.Round(round) // write the msgpack bytes for a block g.latestBlockMsgp = protocol.EncodeMsgp(&cert) - _, err := output.Write(g.latestBlockMsgp) + _, err = output.Write(g.latestBlockMsgp) if err != nil { return err } - g.finishRound(numTxnForBlock) + g.finishRound() return nil } @@ -520,37 +595,71 @@ func getAppTxOptions() []interface{} { // ---- Transaction Generation (Pay/Asset/Apps) ---- -func (g *generator) generateTransaction(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { +func (g *generator) generateSignedTxns(round uint64, intra uint64) ([]txn.SignedTxn, []txn.ApplyData, uint64 /* nextIntra */, error) { + // TODO: return the number of transactions generated instead of updating intra!!! selection, err := weightedSelection(g.transactionWeights, getTransactionOptions(), paymentTx) if err != nil { - return transactions.SignedTxn{}, transactions.ApplyData{}, err + return nil, nil, intra, err } + var signedTxns []txn.SignedTxn + var ads []txn.ApplyData + var nextIntra uint64 + var expectedID uint64 switch selection { case paymentTx: - return g.generatePaymentTxn(round, intra) + var signedTxn txn.SignedTxn + var ad txn.ApplyData + signedTxn, ad, nextIntra, err = g.generatePaymentTxn(round, intra) + signedTxns = []txn.SignedTxn{signedTxn} + ads = []txn.ApplyData{ad} case assetTx: - return g.generateAssetTxn(round, intra) + var signedTxn txn.SignedTxn + var ad txn.ApplyData + signedTxn, ad, nextIntra, expectedID, err = g.generateAssetTxn(round, intra) + signedTxns = []txn.SignedTxn{signedTxn} + ads = []txn.ApplyData{ad} case applicationTx: - return g.generateAppTxn(round, intra) + signedTxns, ads, nextIntra, expectedID, err = g.generateAppTxn(round, intra) default: - return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("no generator available for %s", selection) + return nil, nil, intra, fmt.Errorf("no generator available for %s", selection) + } + + if err != nil { + return nil, nil, intra, fmt.Errorf("error generating transaction: %w", err) + } + + if len(signedTxns) == 0 { + return nil, nil, intra, fmt.Errorf("no transactions generated") } + + for i := range signedTxns { + g.latestPaysetWithExpectedID = append( + g.latestPaysetWithExpectedID, + txnWithExpectedID{ + expectedID: expectedID, + signedTxn: &signedTxns[i], + intra: intra, + nextIntra: nextIntra, + }, + ) + } + return signedTxns, ads, nextIntra, nil } // ---- 1. Pay Transactions ---- // generatePaymentTxn creates a new payment transaction. The sender is always a genesis account, the receiver is random, // or a new account. -func (g *generator) generatePaymentTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { +func (g *generator) generatePaymentTxn(round uint64, intra uint64) (txn.SignedTxn, txn.ApplyData, uint64 /* nextIntra */, error) { selection, err := weightedSelection(g.payTxWeights, getPaymentTxOptions(), paymentPayTx) if err != nil { - return transactions.SignedTxn{}, transactions.ApplyData{}, err + return txn.SignedTxn{}, txn.ApplyData{}, intra, err } return g.generatePaymentTxnInternal(selection.(TxTypeID), round, intra) } -func (g *generator) generatePaymentTxnInternal(selection TxTypeID, round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { +func (g *generator) generatePaymentTxnInternal(selection TxTypeID, round uint64, intra uint64) (txn.SignedTxn, txn.ApplyData, uint64 /* nextIntra */, error) { defer g.recordData(track(selection)) minBal := g.params.MinBalance @@ -586,36 +695,35 @@ func (g *generator) generatePaymentTxnInternal(selection TxTypeID, round uint64, g.numPayments++ - txn := g.makePaymentTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}) - return signTxn(txn), transactions.ApplyData{}, nil + transaction := g.makePaymentTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}) + return signTxn(transaction), txn.ApplyData{}, intra + 1, nil } // ---- 2. Asset Transactions ---- -func (g *generator) generateAssetTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { +func (g *generator) generateAssetTxn(round uint64, intra uint64) (txn.SignedTxn, txn.ApplyData, uint64 /* nextIntra */, uint64 /* assetID */, error) { start := time.Now() selection, err := weightedSelection(g.assetTxWeights, getAssetTxOptions(), assetXfer) if err != nil { - return transactions.SignedTxn{}, transactions.ApplyData{}, err + return txn.SignedTxn{}, txn.ApplyData{}, intra, 0, err } - actual, txn := g.generateAssetTxnInternal(selection.(TxTypeID), round, intra) + actual, transaction, assetID := g.generateAssetTxnInternal(selection.(TxTypeID), round, intra) defer g.recordData(actual, start) - // TODO: shouldn't we just return an error? - if txn.Type == "" { + if transaction.Type == "" { fmt.Println("Empty asset transaction.") os.Exit(1) } - return signTxn(txn), transactions.ApplyData{}, nil + return signTxn(transaction), txn.ApplyData{}, intra + 1, assetID, nil } -func (g *generator) generateAssetTxnInternal(txType TxTypeID, round uint64, intra uint64) (actual TxTypeID, txn transactions.Transaction) { +func (g *generator) generateAssetTxnInternal(txType TxTypeID, round uint64, intra uint64) (actual TxTypeID, txn txn.Transaction, assetID uint64) { return g.generateAssetTxnInternalHint(txType, round, intra, 0, nil) } -func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, intra uint64, hintIndex uint64, hint *assetData) (actual TxTypeID, txn transactions.Transaction) { +func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, intra uint64, hintIndex uint64, hint *assetData) (actual TxTypeID, txn txn.Transaction, assetID uint64) { actual = txType // If there are no assets the next operation needs to be a create. numAssets := uint64(len(g.assets)) @@ -630,7 +738,7 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, senderAcct := indexToAccount(senderIndex) total := assetTotal - assetID := g.txnCounter + intra + 1 + assetID = g.txnCounter + intra + 1 assetName := fmt.Sprintf("asset #%d", assetID) txn = g.makeAssetCreateTxn(g.makeTxnHeader(senderAcct, round, intra), total, false, assetName) // Compute asset ID and initialize holdings @@ -654,7 +762,7 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, assetIndex = hintIndex asset = hint } else { - assetIndex = rand.Uint64()%numAssets + assetIndex = rand.Uint64() % numAssets asset = g.assets[assetIndex] } @@ -669,7 +777,8 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, senderIndex = asset.creator creator := indexToAccount(senderIndex) - txn = g.makeAssetDestroyTxn(g.makeTxnHeader(creator, round, intra), asset.assetID) + assetID = asset.assetID + txn = g.makeAssetDestroyTxn(g.makeTxnHeader(creator, round, intra), assetID) // Remove asset by moving the last element to the deleted index then trimming the slice. g.assets[assetIndex] = g.assets[numAssets-1] @@ -689,7 +798,8 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, exists = asset.holders[senderIndex] != nil } account := indexToAccount(senderIndex) - txn = g.makeAssetAcceptanceTxn(g.makeTxnHeader(account, round, intra), asset.assetID) + assetID = asset.assetID + txn = g.makeAssetAcceptanceTxn(g.makeTxnHeader(account, round, intra), assetID) holding := assetHolding{ acctIndex: senderIndex, @@ -712,14 +822,15 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, receiver := indexToAccount(asset.holdings[receiverArrayIndex].acctIndex) amount := uint64(10) - txn = g.makeAssetTransferTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}, asset.assetID) + assetID = asset.assetID + txn = g.makeAssetTransferTxn(g.makeTxnHeader(sender, round, intra), receiver, amount, basics.Address{}, assetID) if asset.holdings[0].balance < amount { - fmt.Printf("\n\ncreator doesn't have enough funds for asset %d\n\n", asset.assetID) + fmt.Printf("\n\ncreator doesn't have enough funds for asset %d\n\n", assetID) os.Exit(1) } if g.balances[asset.holdings[0].acctIndex] < g.params.MinTxnFee { - fmt.Printf("\n\ncreator doesn't have enough funds for transaction %d\n\n", asset.assetID) + fmt.Printf("\n\ncreator doesn't have enough funds for transaction %d\n\n", assetID) os.Exit(1) } @@ -741,8 +852,9 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, closeToAcctIndex := asset.holdings[0].acctIndex closeToAcct := indexToAccount(closeToAcctIndex) + assetID = asset.assetID txn = g.makeAssetTransferTxn( - g.makeTxnHeader(sender, round, intra), closeToAcct, 0, closeToAcct, asset.assetID) + g.makeTxnHeader(sender, round, intra), closeToAcct, 0, closeToAcct, assetID) asset.holdings[0].balance += asset.holdings[closeIndex].balance @@ -763,110 +875,242 @@ func (g *generator) generateAssetTxnInternalHint(txType TxTypeID, round uint64, fmt.Printf("\n\nthe sender account does not have enough algos for the transfer. idx %d, asset transaction type %v, num %d\n\n", senderIndex, actual, g.reportData[actual].GenerationCount) os.Exit(1) } + + if assetID == 0 { + fmt.Printf("\n\nthis should never happen: assetID is 0 but should have been set by \ngenerateAssetTxnInternalHint(txType=%s, round=%d, intra=%d, hintIndex=%d, hintIsNil=%t)\nactual=%s\n\n", + txType, round, intra, hintIndex, hint == nil, actual) + os.Exit(1) + } + g.balances[senderIndex] -= txn.Fee.ToUint64() + return } // ---- 3. App Transactions ---- -func (g *generator) generateAppTxn(round uint64, intra uint64) (transactions.SignedTxn, transactions.ApplyData, error) { +func (g *generator) generateAppTxn(round uint64, intra uint64) ([]txn.SignedTxn, []txn.ApplyData, uint64 /* nextIntra */, uint64 /* appID */, error) { start := time.Now() selection, err := weightedSelection(g.appTxWeights, getAppTxOptions(), appSwapCall) if err != nil { - return transactions.SignedTxn{}, transactions.ApplyData{}, err + return nil, nil, intra, 0, err } - actual, txn, err := g.generateAppCallInternal(selection.(TxTypeID), round, intra, 0, nil) + actual, signedTxns, appID, err := g.generateAppCallInternal(selection.(TxTypeID), round, intra, nil) if err != nil { - return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("unexpected error received from generateAppCallInternal(): %w", err) + return nil, nil, intra, appID, fmt.Errorf("unexpected error received from generateAppCallInternal(): %w", err) } - if txn.Type == "" { - return transactions.SignedTxn{}, transactions.ApplyData{}, fmt.Errorf("missing transaction type for app transaction") + + if _, ok := effects[actual]; ok { + txCount, err := g.countAndRecordEffects(actual, start) + intra += txCount + if err != nil { + return nil, nil, intra, appID, fmt.Errorf("failed to record app transaction %s: %w", actual, err) + } + } else { // no effects for actual, so exactly 1 transaction + g.recordData(actual, start) + intra++ + } + + ads := make([]txn.ApplyData, len(signedTxns)) + for i := range signedTxns { + ads[i] = txn.ApplyData{} } - g.recordData(actual, start) - return signTxn(txn), transactions.ApplyData{}, nil + return signedTxns, ads, intra, appID, nil } -func (g *generator) generateAppCallInternal(txType TxTypeID, round, intra, hintIndex uint64, hintApp *appData) (TxTypeID, transactions.Transaction, error) { - actual := txType +// generateAppCallInternal is the main workhorse for generating app transactions. +// Senders are always genesis accounts, to avoid running out of funds. +func (g *generator) generateAppCallInternal(txType TxTypeID, round, intra uint64, hintApp *appData) (TxTypeID, []txn.SignedTxn, uint64 /* appID */, error) { + var senderIndex uint64 + if hintApp != nil { + senderIndex = hintApp.sender + } else { + senderIndex = rand.Uint64() % g.config.NumGenesisAccounts + } + senderAcct := indexToAccount(senderIndex) - isApp, kind, appTx, err := parseAppTxType(txType) + actual, kind, appCallType, appID, err := g.getActualAppCall(txType, senderIndex) if err != nil { - return "", transactions.Transaction{}, err + return "", nil, appID, err } - if !isApp { - return "", transactions.Transaction{}, fmt.Errorf("should be an app but not parsed that way: %v", txType) + if hintApp != nil && hintApp.appID != 0 { + // can only override the appID when non-zero in hintApp + appID = hintApp.appID } - if appTx != appTxTypeCreate { - return "", transactions.Transaction{}, fmt.Errorf("invalid transaction type for app %v", appTx) + // WLOG: the matched cases below are now well-defined thanks to getActualAppCall() + + var signedTxns []txn.SignedTxn + switch appCallType { + case appTxTypeCreate: + appID = g.txnCounter + intra + 1 + signedTxns = g.makeAppCreateTxn(kind, senderAcct, round, intra, appID) + reSignTxns(signedTxns) + + for k := range g.appMap { + if g.appMap[k][appID] != nil { + return "", nil, appID, fmt.Errorf("should never happen! app %d already exists for kind %s", appID, k) + } + if g.pendingAppMap[k][appID] != nil { + return "", nil, appID, fmt.Errorf("should never happen! app %d already pending for kind %s", appID, k) + } + } + + ad := &appData{ + appID: appID, + sender: senderIndex, + kind: kind, + optins: map[uint64]bool{}, + } + + g.pendingAppSlice[kind] = append(g.pendingAppSlice[kind], ad) + g.pendingAppMap[kind][appID] = ad + + case appTxTypeOptin: + signedTxns = g.makeAppOptinTxn(senderAcct, round, intra, kind, appID) + reSignTxns(signedTxns) + if g.pendingAppMap[kind][appID] == nil { + ad := &appData{ + appID: appID, + sender: senderIndex, + kind: kind, + optins: map[uint64]bool{}, + } + g.pendingAppMap[kind][appID] = ad + g.pendingAppSlice[kind] = append(g.pendingAppSlice[kind], ad) + } + g.pendingAppMap[kind][appID].optins[senderIndex] = true + + case appTxTypeCall: + signedTxns = []txn.SignedTxn{ + signTxn(g.makeAppCallTxn(senderAcct, round, intra, appID)), + } + + default: + return "", nil, appID, fmt.Errorf("unimplemented: invalid transaction type <%s> for app %d", appCallType, appID) } - var senderIndex uint64 - if hintApp != nil { - return "", transactions.Transaction{}, fmt.Errorf("not ready for hint app %v", hintApp) + return actual, signedTxns, appID, nil +} + +func (g *generator) getAppData(existing bool, kind appKind, senderIndex, appID uint64) (*appData, bool /* appInMap */, bool /* senderOptedin */) { + var appMapOrPendingAppMap map[appKind]map[uint64]*appData + if existing { + appMapOrPendingAppMap = g.appMap } else { - senderIndex = rand.Uint64() % g.numAccounts + appMapOrPendingAppMap = g.pendingAppMap } - actualAppTx := appTx + ad, ok := appMapOrPendingAppMap[kind][appID] + if !ok { + return nil, false, false + } + if !ad.optins[senderIndex] { + return ad, true, false + } + return ad, true, true +} - numApps := uint64(len(g.apps[kind])) - if numApps == 0 { - actualAppTx = appTxTypeCreate +// getActualAppCall returns the actual transaction type, app kind, app transaction type and appID +// * it returns actual = txType if there aren't any problems (for example create always is kept) +// * it creates the app if the app of the given kind doesn't exist +// * it switches to noopoc instead of optin when already opted into existing apps +// * it switches to create instead of optin when only opted into pending apps +// * it switches to optin when noopoc if not opted in and follows the logic of the optins above +// * the appID is 0 for creates, and otherwise a random appID from the existing apps for the kind +func (g *generator) getActualAppCall(txType TxTypeID, senderIndex uint64) (TxTypeID, appKind, appTxType, uint64 /* appID */, error) { + isApp, kind, appTxType, err := parseAppTxType(txType) + if err != nil { + return "", 0, 0, 0, err + } + if !isApp { + return "", 0, 0, 0, fmt.Errorf("should be an app but not parsed that way: %v", txType) } - var txn transactions.Transaction - if actualAppTx == appTxTypeCreate { - numApps += uint64(len(g.pendingApps[kind])) - senderIndex = numApps % g.config.NumGenesisAccounts - senderAcct := indexToAccount(senderIndex) + // creates get a quick pass: + if appTxType == appTxTypeCreate { + return txType, kind, appTxTypeCreate, 0, nil + } - var approval, clear string - if kind == appKindSwap { - approval, clear = approvalSwap, clearSwap - } else { - approval, clear = approvalBoxes, clearBoxes + numAppsForKind := uint64(len(g.appSlice[kind])) + if numAppsForKind == 0 { + // can't do anything else with the app if it doesn't exist, so must create it first!!! + return getAppTxType(kind, appTxTypeCreate), kind, appTxTypeCreate, 0, nil + } + + if appTxType == appTxTypeOptin { + // pick a random app to optin: + appID := g.appSlice[kind][rand.Uint64()%numAppsForKind].appID + + _, exists, optedIn := g.getAppData(true /* existing */, kind, senderIndex, appID) + if !exists { + return txType, kind, appTxType, appID, fmt.Errorf("should never happen! app %d of kind %s does not exist", appID, kind) } - txn = g.makeAppCreateTxn(senderAcct, round, intra, approval, clear) + if optedIn { + // already optedin, so call the app instead: + return getAppTxType(kind, appTxTypeCall), kind, appTxTypeCall, appID, nil + } - appID := g.txnCounter + intra + 1 - holding := &appHolding{appIndex: appID} - ad := &appData{ - appID: appID, - creator: senderIndex, - kind: kind, - holdings: []*appHolding{holding}, - holders: map[uint64]*appHolding{senderIndex: holding}, + _, _, optedInPending := g.getAppData(false /* pending */, kind, senderIndex, appID) + if optedInPending { + // about to get opted in, but can't optin twice or call yet, so create: + return getAppTxType(kind, appTxTypeCreate), kind, appTxTypeCreate, appID, nil } - g.pendingApps[kind] = append(g.pendingApps[kind], ad) + // not opted in or pending, so optin: + return txType, kind, appTxType, appID, nil } - // account := indexToAccount(senderIndex) - // txn = g.makeAppCallTxn(account, round, intra, round, approval, clear) + if appTxType != appTxTypeCall { + return "", 0, 0, 0, fmt.Errorf("unimplemented transaction type for app %s from %s", appTxType, txType) + } + // WLOG appTxTypeCall: - if g.balances[senderIndex] < g.params.MinTxnFee { - return "", transactions.Transaction{}, fmt.Errorf("the sender account does not have enough algos for the app call. idx %d, app transaction type %v, num %d\n\n", senderIndex, txType, g.reportData[txType].GenerationCount) + numAppsOptedin := uint64(len(g.accountAppOptins[kind][senderIndex])) + if numAppsOptedin == 0 { + // try again calling recursively but attempting to optin: + return g.getActualAppCall(getAppTxType(kind, appTxTypeOptin), senderIndex) } - g.balances[senderIndex] -= g.params.MinTxnFee + // WLOG appTxTypeCall with available optins: - return actual, txn, nil + appID := g.accountAppOptins[kind][senderIndex][rand.Uint64()%numAppsOptedin] + return txType, kind, appTxType, appID, nil } -// ---- miscellaneous ---- +// ---- metric data recorders ---- func track(id TxTypeID) (TxTypeID, time.Time) { return id, time.Now() } func (g *generator) recordData(id TxTypeID, start time.Time) { + g.recordOccurrences(id, 1, start) +} + +func (g *generator) recordOccurrences(id TxTypeID, count uint64, start time.Time) { + g.latestData[id] += count data := g.reportData[id] - data.GenerationCount++ + data.GenerationCount += count data.GenerationTime += time.Since(start) g.reportData[id] = data } +func (g *generator) countAndRecordEffects(id TxTypeID, start time.Time) (uint64, error) { + g.recordData(id, start) // this may be a bug!!! + count := uint64(1) + if consequences, ok := effects[id]; ok { + for _, effect := range consequences { + count += effect.count + g.recordOccurrences(effect.txType, effect.count, start) + } + return count, nil + } + return 1, fmt.Errorf("no effects for TxTypeId %v", id) +} + +// ---- miscellaneous ---- + func (g *generator) txnForRound(round uint64) uint64 { // There are no transactions in the 0th round if round == 0 { @@ -875,10 +1119,24 @@ func (g *generator) txnForRound(round uint64) uint64 { return g.config.TxnPerBlock } -// finishRound tells the generator it can apply any pending state. -func (g *generator) finishRound(txnCount uint64) { - g.txnCounter += txnCount +// startRound updates the generator's txnCounter based on the latest block header. +// It is assumed that g.round has already been incremented in finishRound() +func (g *generator) startRound() error { + if g.round == 0 { + // nothing to do in round 0 + return nil + } + latestHeader, err := g.ledger.BlockHdr(basics.Round(g.round - 1)) + if err != nil { + return fmt.Errorf("Could not obtain block header for round %d: %w", g.round, err) + } + g.txnCounter = latestHeader.TxnCounter + return nil +} + +// finishRound tells the generator it can apply any pending state and updates its round +func (g *generator) finishRound() { g.timestamp += consensusTimeMilli g.round++ @@ -886,24 +1144,181 @@ func (g *generator) finishRound(txnCount uint64) { g.assets = append(g.assets, g.pendingAssets...) g.pendingAssets = nil - // Apply pending apps... - for _, kind := range []appKind{appKindSwap, appKindBoxes} { - g.apps[kind] = append(g.apps[kind], g.pendingApps[kind]...) - g.pendingApps[kind] = nil + g.latestPaysetWithExpectedID = nil + g.latestData = make(map[TxTypeID]uint64) + + for kind, pendingAppSlice := range g.pendingAppSlice { + for _, pendingApp := range pendingAppSlice { + appID := pendingApp.appID + if g.appMap[kind][appID] == nil { + g.appSlice[kind] = append(g.appSlice[kind], pendingApp) + g.appMap[kind][appID] = pendingApp + for sender := range pendingApp.optins { + g.accountAppOptins[kind][sender] = append(g.accountAppOptins[kind][sender], appID) + } + } else { // just union the optins when already exists + for sender := range pendingApp.optins { + g.appMap[kind][appID].optins[sender] = true + g.accountAppOptins[kind][sender] = append(g.accountAppOptins[kind][sender], appID) + } + } + } } + g.resetPendingApps() } -func signTxn(txn transactions.Transaction) transactions.SignedTxn { - stxn := transactions.SignedTxn{ - Sig: crypto.Signature{}, +func signTxn(transaction txn.Transaction) txn.SignedTxn { + stxn := txn.SignedTxn{ Msig: crypto.MultisigSig{}, - Lsig: transactions.LogicSig{}, - Txn: txn, + Lsig: txn.LogicSig{}, + Txn: transaction, AuthAddr: basics.Address{}, } + addSignature(&stxn) + + return stxn +} + +func addSignature(stxn *txn.SignedTxn) { + stxn.Sig = crypto.Signature{} // TODO: Would it be useful to generate a random signature? stxn.Sig[32] = 50 +} - return stxn +func reSignTxns(signedTxns []txn.SignedTxn) { + for i := range signedTxns { + addSignature(&signedTxns[i]) + } +} + +func (g *generator) introspectLedgerVsGenerator(roundNumber, intra uint64) (errs []error) { + round := basics.Round(roundNumber) + block, err := g.ledger.Block(round) + if err != nil { + round = err.(ledgercore.ErrNoEntry).Committed + fmt.Printf("WARNING: inconsistent generator v. ledger state. Reset round=%d: %v\n", round, err) + errs = append(errs, err) + } + + ledgerStateDeltas, err := g.ledger.GetStateDeltaForRound(round) + if err != nil { + errs = append(errs, err) + } + + cumulative := make(map[TxTypeID]uint64) + for ttID, data := range g.reportData { + cumulative[ttID] = data.GenerationCount + } + + sum := uint64(0) + for ttID, cnt := range cumulative { + if ttID == genesis { + continue + } + sum += cnt + } + fmt.Print("--------------------\n") + fmt.Printf("roundNumber (generator): %d\n", roundNumber) + fmt.Printf("round (ledger): %d\n", round) + fmt.Printf("g.txnCounter + intra: %d\n", g.txnCounter+intra) + fmt.Printf("block.BlockHeader.TxnCounter: %d\n", block.BlockHeader.TxnCounter) + fmt.Printf("len(g.latestPaysetWithExpectedID): %d\n", len(g.latestPaysetWithExpectedID)) + fmt.Printf("g.latestData: %+v\n", g.latestData) + fmt.Printf("cumuluative : %+v\n", cumulative) + fmt.Printf("all txn sum: %d\n", sum) + fmt.Print("--------------------\n") + + // ---- FROM THE LEDGER: box and createable evidence ---- // + + ledgerBoxEvidenceCount := 0 + ledgerBoxEvidence := make(map[uint64][]uint64) + boxes := ledgerStateDeltas.KvMods + for k := range boxes { + appID, nameIEsender, _ := apps.SplitBoxKey(k) + ledgerBoxEvidence[appID] = append(ledgerBoxEvidence[appID], binary.LittleEndian.Uint64([]byte(nameIEsender))-1) + ledgerBoxEvidenceCount++ + } + + // TODO: can get richer info about app-Creatables from: + // updates.Accts.AppResources + ledgerCreatableAppsEvidence := make(map[uint64]uint64) + for creatableID, creatable := range ledgerStateDeltas.Creatables { + if creatable.Ctype == basics.AppCreatable { + ledgerCreatableAppsEvidence[uint64(creatableID)] = accountToIndex(creatable.Creator) + } + } + fmt.Printf("ledgerBoxEvidenceCount: %d\n", ledgerBoxEvidenceCount) + fmt.Printf("ledgerCreatableAppsEvidence: %d\n", len(ledgerCreatableAppsEvidence)) + + // ---- FROM THE GENERATOR: expected created and optins ---- // + + expectedCreated := map[appKind]map[uint64]uint64{ + appKindBoxes: make(map[uint64]uint64), + appKindSwap: make(map[uint64]uint64), + } + expectedOptins := map[appKind]map[uint64]map[uint64]bool{ + appKindBoxes: make(map[uint64]map[uint64]bool), + appKindSwap: make(map[uint64]map[uint64]bool), + } + + expectedOptinsCount := 0 + for kind, appMap := range g.pendingAppMap { + for appID, ad := range appMap { + if len(ad.optins) > 0 { + expectedOptins[kind][appID] = ad.optins + expectedOptinsCount += len(ad.optins) + } else { + expectedCreated[kind][appID] = ad.sender + } + } + } + fmt.Printf("expectedCreatedCount: %d\n", len(expectedCreated[appKindBoxes])) + fmt.Printf("expectedOptinsCount: %d\n", expectedOptinsCount) + + // ---- COMPARE LEDGER AND GENERATOR EVIDENCE ---- // + + ledgerCreatablesUnexpected := map[uint64]uint64{} + for creatableID, creator := range ledgerCreatableAppsEvidence { + if expectedCreated[appKindSwap][creatableID] != creator && expectedCreated[appKindBoxes][creatableID] != creator { + ledgerCreatablesUnexpected[creatableID] = creator + } + } + generatorExpectedCreatablesNotFound := map[uint64]uint64{} + for creatableID, creator := range expectedCreated[appKindBoxes] { + if ledgerCreatableAppsEvidence[creatableID] != creator { + generatorExpectedCreatablesNotFound[creatableID] = creator + } + } + + ledgerBoxOptinsUnexpected := map[uint64][]uint64{} + for appId, boxOptins := range ledgerBoxEvidence { + for _, optin := range boxOptins { + if _, ok := expectedOptins[appKindBoxes][appId][optin]; !ok { + ledgerBoxOptinsUnexpected[appId] = append(ledgerBoxOptinsUnexpected[appId], optin) + } + } + } + + generatorExpectedOptinsNotFound := map[uint64][]uint64{} + for appId, appOptins := range expectedOptins[appKindBoxes] { + for optin := range appOptins { + missing := true + for _, boxOptin := range ledgerBoxEvidence[appId] { + if boxOptin == optin { + missing = false + break + } + } + if missing { + generatorExpectedOptinsNotFound[appId] = append(generatorExpectedOptinsNotFound[appId], optin) + } + } + } + + fmt.Printf("ledgerCreatablesUnexpected: %+v\n", ledgerCreatablesUnexpected) + fmt.Printf("generatorExpectedCreatablesNotFound: %+v\n", generatorExpectedCreatablesNotFound) + fmt.Printf("ledgerBoxOptinsUnexpected: %+v\n", ledgerBoxOptinsUnexpected) + fmt.Printf("expectedOptinsNotFound: %+v\n", generatorExpectedOptinsNotFound) + return errs } diff --git a/tools/block-generator/generator/generate_test.go b/tools/block-generator/generator/generate_test.go index 60d45fe857..5a672efabe 100644 --- a/tools/block-generator/generator/generate_test.go +++ b/tools/block-generator/generator/generate_test.go @@ -22,10 +22,13 @@ import ( "net/http" "net/http/httptest" "testing" + "time" + "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/go-algorand/test/partitiontest" @@ -42,7 +45,7 @@ func makePrivateGenerator(t *testing.T, round uint64, genesis bookkeeping.Genesi AssetCreateFraction: 1.0, } cfg.validateWithDefaults(true) - publicGenerator, err := MakeGenerator(round, genesis, cfg) + publicGenerator, err := MakeGenerator(round, genesis, cfg, true) require.NoError(t, err) return publicGenerator.(*generator) } @@ -66,7 +69,8 @@ func TestAssetXferNoAssetsOverride(t *testing.T) { g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) // First asset transaction must create. - actual, txn := g.generateAssetTxnInternal(assetXfer, 1, 0) + actual, txn, assetID := g.generateAssetTxnInternal(assetXfer, 1, 0) + require.NotEqual(t, 0, assetID) require.Equal(t, assetCreate, actual) require.Equal(t, protocol.AssetConfigTx, txn.Type) require.Len(t, g.assets, 0) @@ -78,12 +82,13 @@ func TestAssetXferNoAssetsOverride(t *testing.T) { func TestAssetXferOneHolderOverride(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() // Transfer converted to optin if there is only 1 holder. - actual, txn := g.generateAssetTxnInternal(assetXfer, 2, 0) + actual, txn, assetID := g.generateAssetTxnInternal(assetXfer, 2, 0) + require.NotEqual(t, 0, assetID) require.Equal(t, assetOptin, actual) require.Equal(t, protocol.AssetTransferTx, txn.Type) require.Len(t, g.assets, 1) @@ -95,12 +100,13 @@ func TestAssetXferOneHolderOverride(t *testing.T) { func TestAssetCloseCreatorOverride(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() // Instead of closing the creator, optin a new account - actual, txn := g.generateAssetTxnInternal(assetClose, 2, 0) + actual, txn, assetID := g.generateAssetTxnInternal(assetClose, 2, 0) + require.NotEqual(t, 0, assetID) require.Equal(t, assetOptin, actual) require.Equal(t, protocol.AssetTransferTx, txn.Type) require.Len(t, g.assets, 1) @@ -112,29 +118,32 @@ func TestAssetCloseCreatorOverride(t *testing.T) { func TestAssetOptinEveryAccountOverride(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() // Opt all the accounts in, this also verifies that no account is opted in twice var txn transactions.Transaction var actual TxTypeID + var assetID uint64 for i := 2; uint64(i) <= g.numAccounts; i++ { - actual, txn = g.generateAssetTxnInternal(assetOptin, 2, uint64(1+i)) + actual, txn, assetID = g.generateAssetTxnInternal(assetOptin, 2, uint64(1+i)) + require.NotEqual(t, 0, assetID) require.Equal(t, assetOptin, actual) require.Equal(t, protocol.AssetTransferTx, txn.Type) require.Len(t, g.assets, 1) require.Len(t, g.assets[0].holdings, i) require.Len(t, g.assets[0].holders, i) } - g.finishRound(2) + g.finishRound() // All accounts have opted in require.Equal(t, g.numAccounts, uint64(len(g.assets[0].holdings))) // The next optin closes instead - actual, txn = g.generateAssetTxnInternal(assetOptin, 3, 0) - g.finishRound(3) + actual, txn, assetID = g.generateAssetTxnInternal(assetOptin, 3, 0) + require.Greater(t, assetID, uint64(0)) + g.finishRound() require.Equal(t, assetClose, actual) require.Equal(t, protocol.AssetTransferTx, txn.Type) require.Len(t, g.assets, 1) @@ -145,17 +154,18 @@ func TestAssetOptinEveryAccountOverride(t *testing.T) { func TestAssetDestroyWithHoldingsOverride(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() g.generateAssetTxnInternal(assetOptin, 2, 0) - g.finishRound(2) + g.finishRound() g.generateAssetTxnInternal(assetXfer, 3, 0) - g.finishRound(3) + g.finishRound() require.Len(t, g.assets[0].holdings, 2) require.Len(t, g.assets[0].holders, 2) - actual, txn := g.generateAssetTxnInternal(assetDestroy, 4, 0) + actual, txn, assetID := g.generateAssetTxnInternal(assetDestroy, 4, 0) + require.NotEqual(t, 0, assetID) require.Equal(t, assetClose, actual) require.Equal(t, protocol.AssetTransferTx, txn.Type) require.Len(t, g.assets, 1) @@ -166,72 +176,263 @@ func TestAssetDestroyWithHoldingsOverride(t *testing.T) { func TestAssetTransfer(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() g.generateAssetTxnInternal(assetOptin, 2, 0) - g.finishRound(2) + g.finishRound() g.generateAssetTxnInternal(assetXfer, 3, 0) - g.finishRound(3) - require.Greater(t, g.assets[0].holdings[1].balance, uint64(0)) + g.finishRound() + require.NotEqual(t, g.assets[0].holdings[1].balance, uint64(0)) } func TestAssetDestroy(t *testing.T) { partitiontest.PartitionTest(t) g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) - g.finishRound(0) + g.finishRound() g.generateAssetTxnInternal(assetCreate, 1, 0) - g.finishRound(1) + g.finishRound() require.Len(t, g.assets, 1) - actual, txn := g.generateAssetTxnInternal(assetDestroy, 2, 0) + actual, txn, assetID := g.generateAssetTxnInternal(assetDestroy, 2, 0) + require.NotEqual(t, 0, assetID) require.Equal(t, assetDestroy, actual) require.Equal(t, protocol.AssetConfigTx, txn.Type) require.Len(t, g.assets, 0) } +type assembledPrograms struct { + boxesApproval []byte + boxesClear []byte + swapsApproval []byte + swapsClear []byte +} + +func assembleApps(t *testing.T) assembledPrograms { + t.Helper() + + ap := assembledPrograms{} + + ops, err := logic.AssembleString(approvalBoxes) + ap.boxesApproval = ops.Program + require.NoError(t, err) + ops, err = logic.AssembleString(clearBoxes) + ap.boxesClear = ops.Program + require.NoError(t, err) + + ops, err = logic.AssembleString(approvalSwap) + ap.swapsApproval = ops.Program + require.NoError(t, err) + ops, err = logic.AssembleString(clearSwap) + ap.swapsClear = ops.Program + require.NoError(t, err) + + return ap +} + func TestAppCreate(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) + assembled := assembleApps(t) + + round, intra := uint64(1337), uint64(0) + hint := appData{sender: 7} // app call transaction creating appBoxes - actual, txn, err := g.generateAppCallInternal(appBoxesCreate, 1, 0, 0, nil) + actual, sgnTxns, appID, err := g.generateAppCallInternal(appBoxesCreate, round, intra, &hint) + _ = appID require.NoError(t, err) require.Equal(t, appBoxesCreate, actual) - require.Equal(t, protocol.ApplicationCallTx, txn.Type) - require.Len(t, g.apps, 0) - require.Len(t, g.pendingApps, 1) - require.Len(t, g.pendingApps[appKindBoxes], 1) - require.Len(t, g.pendingApps[appKindSwap], 0) - require.Len(t, g.pendingApps[appKindBoxes][0].holdings, 1) - require.Len(t, g.pendingApps[appKindBoxes][0].holders, 1) - ad := *g.pendingApps[appKindBoxes][0] - holding := *ad.holdings[0] - require.Equal(t, holding, *ad.holders[0]) - require.Equal(t, uint64(1001), holding.appIndex) - require.Equal(t, ad.appID, holding.appIndex) + + require.Len(t, sgnTxns, 2) + createTxn := sgnTxns[0].Txn + + require.Equal(t, indexToAccount(hint.sender), createTxn.Sender) + require.Equal(t, protocol.ApplicationCallTx, createTxn.Type) + require.Equal(t, basics.AppIndex(0), createTxn.ApplicationCallTxnFields.ApplicationID) + require.Equal(t, assembled.boxesApproval, createTxn.ApplicationCallTxnFields.ApprovalProgram) + require.Equal(t, assembled.boxesClear, createTxn.ApplicationCallTxnFields.ClearStateProgram) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumByteSlice) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumUint) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumByteSlice) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumUint) + require.Equal(t, transactions.NoOpOC, createTxn.ApplicationCallTxnFields.OnCompletion) + + require.Len(t, g.pendingAppSlice[appKindBoxes], 1) + require.Len(t, g.pendingAppSlice[appKindSwap], 0) + require.Len(t, g.pendingAppMap[appKindBoxes], 1) + require.Len(t, g.pendingAppMap[appKindSwap], 0) + ad := g.pendingAppSlice[appKindBoxes][0] + require.Equal(t, ad, g.pendingAppMap[appKindBoxes][ad.appID]) + require.Equal(t, hint.sender, ad.sender) require.Equal(t, appKindBoxes, ad.kind) + optins := ad.optins + require.Len(t, optins, 0) + + paySiblingTxn := sgnTxns[1].Txn + require.Equal(t, protocol.PaymentTx, paySiblingTxn.Type) // app call transaction creating appSwap - actual, txn, err = g.generateAppCallInternal(appSwapCreate, 1, 0, 0, nil) + intra = 1 + actual, sgnTxns, appID, err = g.generateAppCallInternal(appSwapCreate, round, intra, &hint) + _ = appID require.NoError(t, err) require.Equal(t, appSwapCreate, actual) - require.Equal(t, protocol.ApplicationCallTx, txn.Type) - require.Len(t, g.apps, 0) - require.Len(t, g.pendingApps, 2) - require.Len(t, g.pendingApps[appKindBoxes], 1) - require.Len(t, g.pendingApps[appKindSwap], 1) - require.Len(t, g.pendingApps[appKindSwap][0].holdings, 1) - require.Len(t, g.pendingApps[appKindSwap][0].holders, 1) - ad = *g.pendingApps[appKindSwap][0] - holding = *ad.holdings[0] - require.Equal(t, holding, *ad.holders[0]) - require.Equal(t, uint64(1001), holding.appIndex) - require.Equal(t, ad.appID, holding.appIndex) + + require.Len(t, sgnTxns, 1) + createTxn = sgnTxns[0].Txn + + require.Equal(t, protocol.ApplicationCallTx, createTxn.Type) + require.Equal(t, indexToAccount(hint.sender), createTxn.Sender) + require.Equal(t, basics.AppIndex(0), createTxn.ApplicationCallTxnFields.ApplicationID) + require.Equal(t, assembled.swapsApproval, createTxn.ApplicationCallTxnFields.ApprovalProgram) + require.Equal(t, assembled.swapsClear, createTxn.ApplicationCallTxnFields.ClearStateProgram) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumByteSlice) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumUint) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumByteSlice) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumUint) + require.Equal(t, transactions.NoOpOC, createTxn.ApplicationCallTxnFields.OnCompletion) + + require.Len(t, g.pendingAppSlice[appKindBoxes], 1) + require.Len(t, g.pendingAppSlice[appKindSwap], 1) + require.Len(t, g.pendingAppMap[appKindBoxes], 1) + require.Len(t, g.pendingAppMap[appKindSwap], 1) + ad = g.pendingAppSlice[appKindSwap][0] + require.Equal(t, ad, g.pendingAppMap[appKindSwap][ad.appID]) + require.Equal(t, hint.sender, ad.sender) require.Equal(t, appKindSwap, ad.kind) + optins = ad.optins + require.Len(t, optins, 0) +} + +func TestAppBoxesOptin(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) + assembled := assembleApps(t) + + round, intra := uint64(1337), uint64(0) + + hint := appData{sender: 7} + + // app call transaction opting into boxes gets replaced by creating appBoxes + actual, sgnTxns, appID, err := g.generateAppCallInternal(appBoxesOptin, round, intra, &hint) + _ = appID + require.NoError(t, err) + require.Equal(t, appBoxesCreate, actual) + + require.Len(t, sgnTxns, 2) + createTxn := sgnTxns[0].Txn + + require.Equal(t, protocol.ApplicationCallTx, createTxn.Type) + require.Equal(t, indexToAccount(hint.sender), createTxn.Sender) + require.Equal(t, basics.AppIndex(0), createTxn.ApplicationCallTxnFields.ApplicationID) + require.Equal(t, assembled.boxesApproval, createTxn.ApplicationCallTxnFields.ApprovalProgram) + require.Equal(t, assembled.boxesClear, createTxn.ApplicationCallTxnFields.ClearStateProgram) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumByteSlice) + require.Equal(t, uint64(32), createTxn.ApplicationCallTxnFields.GlobalStateSchema.NumUint) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumByteSlice) + require.Equal(t, uint64(8), createTxn.ApplicationCallTxnFields.LocalStateSchema.NumUint) + require.Equal(t, transactions.NoOpOC, createTxn.ApplicationCallTxnFields.OnCompletion) + require.Nil(t, createTxn.ApplicationCallTxnFields.Boxes) + + require.Len(t, g.pendingAppSlice[appKindBoxes], 1) + require.Len(t, g.pendingAppSlice[appKindSwap], 0) + require.Len(t, g.pendingAppMap[appKindBoxes], 1) + require.Len(t, g.pendingAppMap[appKindSwap], 0) + ad := g.pendingAppSlice[appKindBoxes][0] + require.Equal(t, ad, g.pendingAppMap[appKindBoxes][ad.appID]) + require.Equal(t, hint.sender, ad.sender) + require.Equal(t, appKindBoxes, ad.kind) + require.Len(t, ad.optins, 0) + + require.Contains(t, effects, actual) + + paySiblingTxn := sgnTxns[1].Txn + require.Equal(t, protocol.PaymentTx, paySiblingTxn.Type) + + // 2nd attempt to optin (with new sender) doesn't get replaced + g.finishRound() + intra += 1 + hint.sender = 8 + + actual, sgnTxns, appID, err = g.generateAppCallInternal(appBoxesOptin, round, intra, &hint) + _ = appID + require.NoError(t, err) + require.Equal(t, appBoxesOptin, actual) + + require.Len(t, sgnTxns, 2) + pay := sgnTxns[1].Txn + require.Equal(t, protocol.PaymentTx, pay.Type) + require.NotEqual(t, basics.Address{}.String(), pay.Sender.String()) + + createTxn = sgnTxns[0].Txn + require.Equal(t, protocol.ApplicationCallTx, createTxn.Type) + require.Equal(t, indexToAccount(hint.sender), createTxn.Sender) + require.Equal(t, basics.AppIndex(1001), createTxn.ApplicationCallTxnFields.ApplicationID) + require.Equal(t, []byte(nil), createTxn.ApplicationCallTxnFields.ApprovalProgram) + require.Equal(t, []byte(nil), createTxn.ApplicationCallTxnFields.ClearStateProgram) + require.Equal(t, basics.StateSchema{}, createTxn.ApplicationCallTxnFields.GlobalStateSchema) + require.Equal(t, basics.StateSchema{}, createTxn.ApplicationCallTxnFields.LocalStateSchema) + require.Equal(t, transactions.OptInOC, createTxn.ApplicationCallTxnFields.OnCompletion) + require.Len(t, createTxn.ApplicationCallTxnFields.Boxes, 1) + require.Equal(t, crypto.Digest(pay.Sender).ToSlice(), createTxn.ApplicationCallTxnFields.Boxes[0].Name) + + require.Len(t, g.pendingAppSlice[appKindBoxes], 1) + require.Len(t, g.pendingAppSlice[appKindSwap], 0) + require.Len(t, g.pendingAppMap[appKindBoxes], 1) + require.Len(t, g.pendingAppMap[appKindSwap], 0) + ad = g.pendingAppSlice[appKindBoxes][0] + require.Equal(t, ad, g.pendingAppMap[appKindBoxes][ad.appID]) + require.Equal(t, hint.sender, ad.sender) // NOT 8!!! + require.Equal(t, appKindBoxes, ad.kind) + optins := ad.optins + require.Len(t, optins, 1) + require.Contains(t, optins, hint.sender) + + require.Contains(t, effects, actual) + require.Len(t, effects[actual], 2) + require.Equal(t, TxEffect{effectPaymentTxSibling, 1}, effects[actual][0]) + require.Equal(t, TxEffect{effectInnerTx, 2}, effects[actual][1]) + + numTxns, err := g.countAndRecordEffects(actual, time.Now()) + require.NoError(t, err) + require.Equal(t, uint64(4), numTxns) + + // 3rd attempt to optin gets replaced by vanilla app call + g.finishRound() + intra += numTxns + + actual, sgnTxns, appID, err = g.generateAppCallInternal(appBoxesOptin, round, intra, &hint) + _ = appID + require.NoError(t, err) + require.Equal(t, appBoxesCall, actual) + + require.Len(t, sgnTxns, 1) + + createTxn = sgnTxns[0].Txn + require.Equal(t, protocol.ApplicationCallTx, createTxn.Type) + require.Equal(t, indexToAccount(hint.sender), createTxn.Sender) + require.Equal(t, basics.AppIndex(1001), createTxn.ApplicationCallTxnFields.ApplicationID) + require.Equal(t, []byte(nil), createTxn.ApplicationCallTxnFields.ApprovalProgram) + require.Equal(t, []byte(nil), createTxn.ApplicationCallTxnFields.ClearStateProgram) + require.Equal(t, basics.StateSchema{}, createTxn.ApplicationCallTxnFields.GlobalStateSchema) + require.Equal(t, basics.StateSchema{}, createTxn.ApplicationCallTxnFields.LocalStateSchema) + require.Equal(t, transactions.NoOpOC, createTxn.ApplicationCallTxnFields.OnCompletion) + require.Len(t, createTxn.ApplicationCallTxnFields.Boxes, 1) + require.Equal(t, crypto.Digest(pay.Sender).ToSlice(), createTxn.ApplicationCallTxnFields.Boxes[0].Name) + + // no change to app states + require.Len(t, g.pendingAppSlice[appKindBoxes], 0) + require.Len(t, g.pendingAppSlice[appKindSwap], 0) + require.Len(t, g.pendingAppMap[appKindBoxes], 0) + require.Len(t, g.pendingAppMap[appKindSwap], 0) + + require.NotContains(t, effects, actual) } func TestWriteRoundZero(t *testing.T) { @@ -462,3 +663,74 @@ func TestHandlers(t *testing.T) { }) } } + +func TestRecordData(t *testing.T) { + gen := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) + + id := TxTypeID("test") + data, ok := gen.reportData[id] + require.False(t, ok) + + gen.recordData(id, time.Now()) + data, ok = gen.reportData[id] + require.True(t, ok) + require.Equal(t, uint64(1), data.GenerationCount) + + gen.recordData(id, time.Now()) + data, ok = gen.reportData[id] + require.True(t, ok) + require.Equal(t, uint64(2), data.GenerationCount) + +} + +func TestRecordOccurrences(t *testing.T) { + gen := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) + + id := TxTypeID("test") + data, ok := gen.reportData[id] + require.False(t, ok) + + gen.recordOccurrences(id, 100, time.Now()) + data, ok = gen.reportData[id] + require.True(t, ok) + require.Equal(t, uint64(100), data.GenerationCount) + + gen.recordOccurrences(id, 200, time.Now()) + data, ok = gen.reportData[id] + require.True(t, ok) + require.Equal(t, uint64(300), data.GenerationCount) +} + +func TestRecordAppConsequences(t *testing.T) { + g := makePrivateGenerator(t, 0, bookkeeping.Genesis{}) + + txTypeId := TxTypeID("test") + txCount, err := g.countAndRecordEffects(txTypeId, time.Now()) + require.Error(t, err, "no effects for TxTypeId test") + + // recordIncludingEffects always records the root txTypeId + require.Equal(t, uint64(1), txCount) + data, ok := g.reportData[txTypeId] + require.True(t, ok) + require.Equal(t, uint64(1), data.GenerationCount) + require.Len(t, g.reportData, 1) + + txTypeId = appBoxesOptin + txCount, err = g.countAndRecordEffects(txTypeId, time.Now()) + require.NoError(t, err) + require.Equal(t, uint64(4), txCount) + + require.Len(t, g.reportData, 4) + + data, ok = g.reportData[txTypeId] + require.True(t, ok) + require.Equal(t, uint64(1), data.GenerationCount) + + data, ok = g.reportData[effectPaymentTxSibling] + require.True(t, ok) + require.Equal(t, uint64(1), data.GenerationCount) + + data, ok = g.reportData[effectInnerTx] + require.True(t, ok) + require.Equal(t, uint64(2), data.GenerationCount) +} diff --git a/tools/block-generator/generator/generator_types.go b/tools/block-generator/generator/generator_types.go index bb55826ef4..2f4997a4fa 100644 --- a/tools/block-generator/generator/generator_types.go +++ b/tools/block-generator/generator/generator_types.go @@ -24,6 +24,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + txn "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/protocol" ) @@ -40,6 +41,8 @@ type Generator interface { } type generator struct { + verbose bool + config GenerationConfig // payment transaction metadata @@ -49,7 +52,7 @@ type generator struct { numAccounts uint64 // Block stuff - round uint64 + round uint64 txnCounter uint64 prevBlockHash string timestamp int64 @@ -78,11 +81,24 @@ type generator struct { // being created. pendingAssets []*assetData - // apps is a minimal representation of the app holdings - apps map[appKind][]*appData - // pendingApps is used to hold newly created apps so that they are not used before - // being created. - pendingApps map[appKind][]*appData + // pendingAppMap provides a live mapping from appID to appData for each appKind + // for the current round + pendingAppMap map[appKind]map[uint64]*appData + + // pendingAppSlice provides a live slice of appData for each appKind. The reason + // for maintaining both appMap and pendingAppSlice is to enable + // randomly selecting an app to interact with and yet easily access it once + // its identifier is known + pendingAppSlice map[appKind][]*appData + + // appMap and appSlice store the information from their corresponding pending* + // data structures at the end of each round and for the rest of the experiment + appMap map[appKind]map[uint64]*appData + appSlice map[appKind][]*appData + + // accountAppOptins is used to keep track of which accounts have opted into + // and app and enable random selection. + accountAppOptins map[appKind]map[uint64][]uint64 transactionWeights []float32 @@ -92,13 +108,20 @@ type generator struct { // Reporting information from transaction type to data reportData Report + // latestData keeps a count of how many transactions of each + // txType occurred in the current round. + latestData map[TxTypeID]uint64 // ledger ledger *ledger.Ledger - // cache the latest written block + // latestBlockMsgp caches the latest written block latestBlockMsgp []byte + // latestPaysetWithExpectedID provides the ordered payest transactions + // together the expected asset/app IDs (or 0 if not applicable) + latestPaysetWithExpectedID []txnWithExpectedID + roundOffset uint64 } type assetData struct { @@ -112,14 +135,10 @@ type assetData struct { } type appData struct { - appID uint64 - creator uint64 - kind appKind - // Holding at index 0 is the creator. - holdings []*appHolding - // Set of holders in the holdings array for easy reference. - holders map[uint64]*appHolding - // TODO: more data, not sure yet exactly what + appID uint64 + sender uint64 + kind appKind + optins map[uint64]bool } type assetHolding struct { @@ -127,11 +146,6 @@ type assetHolding struct { balance uint64 } -type appHolding struct { - appIndex uint64 - // TODO: more data, not sure yet exactly what -} - // Report is the generation report. type Report map[TxTypeID]TxData @@ -140,3 +154,18 @@ type TxData struct { GenerationTime time.Duration `json:"generation_time_milli"` GenerationCount uint64 `json:"num_generated"` } + +// TxEffect summarizes a txn type count caused by a root transaction. +type TxEffect struct { + txType TxTypeID + count uint64 +} + +// txnWithExpectedID rolls up an expected asset/app ID for non-pay txns +// together with a signedTxn expected to be in the payset. +type txnWithExpectedID struct { + expectedID uint64 + signedTxn *txn.SignedTxn + intra uint64 + nextIntra uint64 +} diff --git a/tools/block-generator/generator/make_transactions.go b/tools/block-generator/generator/make_transactions.go index f7b347e451..cc5ff067ca 100644 --- a/tools/block-generator/generator/make_transactions.go +++ b/tools/block-generator/generator/make_transactions.go @@ -19,6 +19,7 @@ package generator import ( "encoding/binary" + "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" txn "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/txntest" @@ -121,27 +122,20 @@ func (g *generator) makeAssetAcceptanceTxn(header txn.Header, index uint64) txn. // ---- application transactions ---- -func (g *generator) makeAppCreateTxn(sender basics.Address, round, intra uint64, approval, clear string) txn.Transaction { +func (g *generator) makeAppCreateTxn(kind appKind, sender basics.Address, round, intra uint64, futureAppId uint64) []txn.SignedTxn { + var approval, clear string + if kind == appKindSwap { + approval, clear = approvalSwap, clearSwap + } else { + approval, clear = approvalBoxes, clearBoxes + } createTxn := g.makeTestTxn(sender, round, intra) - /* all 0 values but keep around for reference - createTxn.ApplicationID = 0 - createTxn.ApplicationArgs = nil - createTxn.Accounts = nil - createTxn.ForeignApps = nil - createTxn.ForeignAssets = nil - createTxn.Boxes = nil - createTxn.ExtraProgramPages = 0 - */ - createTxn.Type = protocol.ApplicationCallTx createTxn.ApprovalProgram = approval createTxn.ClearStateProgram = clear - // sender opts-in to their own created app - createTxn.OnCompletion = txn.OptInOC - // max out local/global state usage but split // 50% between bytes/uint64 createTxn.LocalStateSchema = basics.StateSchema{ @@ -153,5 +147,91 @@ func (g *generator) makeAppCreateTxn(sender basics.Address, round, intra uint64, NumByteSlice: 32, } - return createTxn.Txn() + createTxFee := g.params.MinTxnFee + senderIndex := accountToIndex(sender) + + // TODO: should check for min balance + g.balances[senderIndex] -= createTxFee + if kind != appKindBoxes { + return txntest.Group(&createTxn) + } + + // also group in a pay txn to fund the app + pstFee := uint64(1_000) + pstAmt := uint64(1_000_000) + + paySibTxn := g.makeTestTxn(sender, round, intra) + paySibTxn.Type = protocol.PaymentTx + paySibTxn.Receiver = basics.AppIndex(futureAppId).Address() + paySibTxn.Fee = basics.MicroAlgos{Raw: pstFee} + paySibTxn.Amount = uint64(pstAmt) + + // TODO: should check for min balance} + g.balances[senderIndex] -= (pstFee + pstAmt) + + return txntest.Group(&createTxn, &paySibTxn) +} + +// makeAppOptinTxn currently only works for the boxes app +func (g *generator) makeAppOptinTxn(sender basics.Address, round, intra uint64, kind appKind, appIndex uint64) []txn.SignedTxn { + if kind != appKindBoxes { + panic("makeAppOptinTxn only works for the boxes app currently") + } + + optInTxn := g.makeTestTxn(sender, round, intra) + /* all 0 values but keep around for reference + optInTxn.ApplicationArgs = nil + optInTxn.ForeignApps = nil + optInTxn.ForeignAssets = nil + optInTxn.ExtraProgramPages = 0 + */ + + optInTxn.Type = protocol.ApplicationCallTx + optInTxn.ApplicationID = basics.AppIndex(appIndex) + optInTxn.OnCompletion = txn.OptInOC + // the first inner sends some algo to the creator: + optInTxn.Accounts = []basics.Address{indexToAccount(g.appMap[kind][appIndex].sender)} + optInTxn.Boxes = []txn.BoxRef{ + {Name: crypto.Digest(sender).ToSlice()}, + } + + // TODO: these may not make sense for the swap optin + + pstFee := uint64(2_000) + pstAmt := uint64(1_000_000) + + paySibTxn := g.makeTestTxn(sender, round, intra) + paySibTxn.Type = protocol.PaymentTx + paySibTxn.Receiver = basics.AppIndex(appIndex).Address() + paySibTxn.Fee = basics.MicroAlgos{Raw: pstFee} + paySibTxn.Amount = uint64(pstAmt) + + senderIndex := accountToIndex(sender) + // TODO: should check for min balance} + // TODO: for the case of boxes, should refund 0.76 algo + g.balances[senderIndex] -= (pstFee + pstAmt) + + return txntest.Group(&optInTxn, &paySibTxn) +} + +// makeAppCallTxn currently only works for the boxes app +func (g *generator) makeAppCallTxn(sender basics.Address, round, intra, appIndex uint64) txn.Transaction { + callTxn := g.makeTestTxn(sender, round, intra) + callTxn.Type = protocol.ApplicationCallTx + callTxn.ApplicationID = basics.AppIndex(appIndex) + callTxn.OnCompletion = txn.NoOpOC // redundant for clarity + callTxn.ApplicationArgs = [][]byte{ + {0xe1, 0xf9, 0x3f, 0x1d}, // the method selector for getting a box + } + + callTxn.Boxes = []txn.BoxRef{ + {Name: crypto.Digest(sender).ToSlice()}, + } + + // TODO: should check for min balance + appCallTxFee := g.params.MinTxnFee + senderIndex := accountToIndex(sender) + g.balances[senderIndex] -= appCallTxFee + + return callTxn.Txn() } diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index c46e926527..2aac4a4552 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -32,7 +32,7 @@ func MakeServer(configFile string, addr string) (*http.Server, Generator) { noOp := func(next http.Handler) http.Handler { return next } - return MakeServerWithMiddleware(0, "", configFile, addr, noOp) + return MakeServerWithMiddleware(0, "", configFile, false, addr, noOp) } // BlocksMiddleware is a middleware for the blocks endpoint. @@ -41,7 +41,7 @@ type BlocksMiddleware func(next http.Handler) http.Handler // MakeServerWithMiddleware allows injecting a middleware for the blocks handler. // This is needed to simplify tests by stopping block production while validation // is done on the data. -func MakeServerWithMiddleware(dbround uint64, genesisFile string, configFile string, addr string, blocksMiddleware BlocksMiddleware) (*http.Server, Generator) { +func MakeServerWithMiddleware(dbround uint64, genesisFile string, configFile string, verbose bool, addr string, blocksMiddleware BlocksMiddleware) (*http.Server, Generator) { cfg, err := initializeConfigFile(configFile) util.MaybeFail(err, "problem loading config file. Use '--config' or create a config file.") var bkGenesis bookkeeping.Genesis @@ -50,7 +50,7 @@ func MakeServerWithMiddleware(dbround uint64, genesisFile string, configFile str // TODO: consider using bkGenesis to set cfg.NumGenesisAccounts and cfg.GenesisAccountInitialBalance util.MaybeFail(err, "Failed to parse genesis file '%s'", genesisFile) } - gen, err := MakeGenerator(dbround, bkGenesis, cfg) + gen, err := MakeGenerator(dbround, bkGenesis, cfg, verbose) util.MaybeFail(err, "Failed to make generator with config file '%s'", configFile) mux := http.NewServeMux() diff --git a/tools/block-generator/generator/teal/poap_boxes.teal b/tools/block-generator/generator/teal/poap_boxes.teal index 445e6c792e..81b52fbeba 100644 --- a/tools/block-generator/generator/teal/poap_boxes.teal +++ b/tools/block-generator/generator/teal/poap_boxes.teal @@ -51,15 +51,15 @@ intc_1 // 2 bnz label4 txn OnCompletion -intc_2 // 1 +intc_2 // 1 == OptInOC == bnz label5 txn NumAppArgs intc_0 // 0 == -bz label6 -b label7 +bz label6 // if have some app args ... continue ... +b label7 // if no app app args: error label6: @@ -71,19 +71,19 @@ label6: pushbytes 0xddc3d103 // 0xddc3d103 txna ApplicationArgs 0 == - bnz label9 + bnz label9 // create an app with approval in arg1 and clear in arg2 and return the new appID pushbytes 0x935e5c25 // 0x935e5c25 txna ApplicationArgs 0 == - bnz label10 + bnz label10 // lets you call an arbitrary app with appid provided in arg1, approval program arg2, and clear program arg3 pushbytes 0xedbb9ab8 // 0xedbb9ab8 txna ApplicationArgs 0 == - bnz label11 + bnz label11 // delete case - pushbytes 0xe1f93f1d // 0xe1f93f1d + pushbytes 0xe1f93f1d // 0xe1f93f1d <--- what we want to call txna ApplicationArgs 0 == bnz label12 @@ -103,55 +103,59 @@ label1: // App Creation global CurrentApplicationID itob concat - b localsfill + b globalset -label15: +label15: // trying to optin with non-zero balance and when app already exists global GroupSize intc_1 // 2 == - assert + assert // must be in a group of 2 gtxn 1 Amount - intc 4 // 2000 + int 1000000 // pay 1 algo + // intc 4 // 2000 == + gtxn 1 Receiver global CurrentApplicationAddress == && assert - gtxn 1 Sender - global CreatorAddress - == - assert - + // we don't want to limit the generator to only allow the creator to be the sender + // so the following is commented out: + // gtxn 1 Sender + // global CreatorAddress + // == + // assert // 2nd txn in the group must be a pymt to the app for .002 Algos from the creator gtxn 1 Fee + intc 4 // 2000 == - assert + assert // fee must be 2000 itxn_begin intc_2 // 1 - itxn_field TypeEnum + itxn_field TypeEnum // payment txn txn Sender itxn_field Receiver pushint 764000 itxn_field Amount global MinTxnFee itxn_field Fee - itxn_submit + itxn_submit // send 0.764 Algos from the app account to the Sender!!! itxn_begin intc_2 // 1 - itxn_field TypeEnum + itxn_field TypeEnum // payment txn global CreatorAddress itxn_field Receiver pushint 4000 itxn_field Amount global MinTxnFee itxn_field Fee - itxn_submit + itxn_submit // send .004 Algos to the creator retsub label5: @@ -159,15 +163,13 @@ label5: txn Sender store 3 - load 3 + load 3 // Sender balance - store 4 + store 4 // SenderBalance load 4 - bnz label14 - callsub label15 - -label14: + bz label7 + callsub label15 // This introduces 2 inner txns bytec_1 // "poap_onboard_count" dup @@ -184,9 +186,11 @@ label14: label16: - load 3 - pushbytes 0x302c // "0," - box_put + load 3 // Sender + // 992 = 1024 - 32 bytes: + byte "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + + box_put // boxes[Sender] = "z" * 992 - our box of size 1024 has MBR 412,100 µA retsub label2: @@ -212,7 +216,7 @@ label3: label4: txn Sender - store 3 + store 3 // 3 -> Sender load 3 box_del @@ -267,7 +271,7 @@ label9: assert gtxn 0 TypeEnum - intc_2 // 1 + intc_2 // 1 // first txn in group is a payment == assert @@ -299,7 +303,7 @@ label9: itxn_begin intc_3 // 6 - itxn_field TypeEnum + itxn_field TypeEnum // app call type intc_0 // 0 itxn_field OnCompletion txna ApplicationArgs 1 @@ -353,7 +357,7 @@ label10: itxn_begin intc_3 // 6 - itxn_field TypeEnum + itxn_field TypeEnum // app call type intc 6 // 4 itxn_field OnCompletion txna ApplicationArgs 1 @@ -407,9 +411,9 @@ label11: txna Assets 0 itxn_field Assets intc_3 // 6 - itxn_field TypeEnum + itxn_field TypeEnum // 6 => app call intc 5 // 5 - itxn_field OnCompletion + itxn_field OnCompletion // 5 => delete application txna ApplicationArgs 1 btoi txnas Applications @@ -432,7 +436,9 @@ label11: label12: - intc_0 // 0 + // intc_0 // 0 + txn Sender + global CurrentApplicationID app_opted_in assert @@ -456,7 +462,7 @@ label13: intc_2 // 1 return -localsfill: +globalset: // 64 bytes byte "Why did the chicken cross the road? The answer has been omitted." @@ -464,82 +470,82 @@ localsfill: txn Sender byte "0000000000000000000000000000000000000000000000000000000000000000" dig 2 - app_local_put + app_global_put txn Sender byte "1111111111111111111111111111111111111111111111111111111111111111" dig 2 - app_local_put + app_global_put txn Sender byte "2222222222222222222222222222222222222222222222222222222222222222" dig 2 - app_local_put + app_global_put txn Sender byte "3333333333333333333333333333333333333333333333333333333333333332" dig 2 - app_local_put + app_global_put txn Sender byte "4444444444444444444444444444444444444444444444444444444444444444" dig 2 - app_local_put + app_global_put txn Sender byte "5555555555555555555555555555555555555555555555555555555555555555" dig 2 - app_local_put + app_global_put txn Sender byte "6666666666666666666666666666666666666666666666666666666666666666" dig 2 - app_local_put + app_global_put txn Sender byte "7777777777777777777777777777777777777777777777777777777777777777" uncover 2 - app_local_put + app_global_put int 1337 txn Sender byte "8888888888888888888888888888888888888888888888888888888888888888" dig 2 - app_local_put + app_global_put txn Sender byte "9999999999999999999999999999999999999999999999999999999999999999" dig 2 - app_local_put + app_global_put txn Sender byte "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" dig 2 - app_local_put + app_global_put txn Sender byte "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" dig 2 - app_local_put + app_global_put txn Sender byte "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" dig 2 - app_local_put + app_global_put txn Sender byte "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" dig 2 - app_local_put + app_global_put txn Sender byte "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" dig 2 - app_local_put + app_global_put txn Sender byte "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" uncover 2 - app_local_put + app_global_put b label13 diff --git a/tools/block-generator/go.mod b/tools/block-generator/go.mod index 8e2eb63afd..12649d09bc 100644 --- a/tools/block-generator/go.mod +++ b/tools/block-generator/go.mod @@ -5,6 +5,7 @@ replace github.com/algorand/go-algorand => ../.. go 1.20 require ( + github.com/algorand/avm-abi v0.2.0 github.com/algorand/go-algorand v0.0.0-00010101000000-000000000000 github.com/algorand/go-codec/codec v1.1.10 github.com/algorand/go-deadlock v0.2.2 @@ -16,7 +17,6 @@ require ( require ( github.com/DataDog/zstd v1.5.2 // indirect - github.com/algorand/avm-abi v0.2.0 // indirect github.com/algorand/falcon v0.1.0 // indirect github.com/algorand/go-sumhash v0.1.0 // indirect github.com/algorand/msgp v1.1.55 // indirect diff --git a/tools/block-generator/run_runner.py b/tools/block-generator/run_runner.py index a95474d834..914a38d467 100644 --- a/tools/block-generator/run_runner.py +++ b/tools/block-generator/run_runner.py @@ -172,7 +172,7 @@ def main(): --conduit-binary "{args.conduit_binary}" \\ --report-directory {args.report_directory} \\ --test-duration {args.test_duration} \\ ---log-level trace \\ +--conduit-log-level trace \\ --postgres-connection-string "host=localhost user=algorand password=algorand dbname={args.pg_database} port={args.pg_port} sslmode=disable" \\ --scenario {args.scenario} {DBS + NL + '--reset-db' if args.reset_db else ''}""" if args.skip_runner: diff --git a/tools/block-generator/run_runner.sh b/tools/block-generator/run_runner.sh index 5e83966318..236d7b2bbb 100755 --- a/tools/block-generator/run_runner.sh +++ b/tools/block-generator/run_runner.sh @@ -54,7 +54,7 @@ $(dirname "$0")/block-generator runner \ --conduit-binary "$CONDUIT_BINARY" \ --report-directory $OUTPUT \ --test-duration 30s \ - --log-level trace \ + --conduit-log-level trace \ --postgres-connection-string "host=localhost user=algorand password=algorand dbname=generator_db port=15432 sslmode=disable" \ --scenario ${SCENARIO} \ --reset-db diff --git a/tools/block-generator/run_tests.sh b/tools/block-generator/run_tests.sh index fcfc7279e0..48c1490b40 100755 --- a/tools/block-generator/run_tests.sh +++ b/tools/block-generator/run_tests.sh @@ -92,6 +92,6 @@ echo "Log Level: $LOG_LEVEL" -d "$DURATION" \ -c "$CONNECTION_STRING" \ --report-directory "$REPORT_DIR" \ - --log-level "$LOG_LEVEL" \ + --conduit-log-level "$LOG_LEVEL" \ --reset-report-dir diff --git a/tools/block-generator/runner/run.go b/tools/block-generator/runner/run.go index d4604f473c..2616d5dfe4 100644 --- a/tools/block-generator/runner/run.go +++ b/tools/block-generator/runner/run.go @@ -51,7 +51,8 @@ type Args struct { PostgresConnectionString string CPUProfilePath string RunDuration time.Duration - LogLevel string + RunnerVerbose bool + ConduitLogLevel string ReportDirectory string ResetReportDir bool RunValidation bool @@ -142,7 +143,7 @@ func (r *Args) run() error { // Start services algodNet := fmt.Sprintf("localhost:%d", 11112) metricsNet := fmt.Sprintf("localhost:%d", r.MetricsPort) - generatorShutdownFunc, _ := startGenerator(r.Path, nextRound, r.GenesisFile, algodNet, blockMiddleware) + generatorShutdownFunc, _ := startGenerator(r.Path, nextRound, r.GenesisFile, r.RunnerVerbose, algodNet, blockMiddleware) defer func() { // Shutdown generator. if err := generatorShutdownFunc(); err != nil { @@ -162,7 +163,7 @@ func (r *Args) run() error { } defer f.Close() - conduitConfig := config{r.LogLevel, logfile, + conduitConfig := config{r.ConduitLogLevel, logfile, fmt.Sprintf(":%d", r.MetricsPort), algodNet, r.PostgresConnectionString, } @@ -407,9 +408,9 @@ func (r *Args) runTest(report *os.File, metricsURL string, generatorURL string) } // startGenerator starts the generator server. -func startGenerator(configFile string, dbround uint64, genesisFile string, addr string, blockMiddleware func(http.Handler) http.Handler) (func() error, generator.Generator) { +func startGenerator(configFile string, dbround uint64, genesisFile string, verbose bool, addr string, blockMiddleware func(http.Handler) http.Handler) (func() error, generator.Generator) { // Start generator. - server, generator := generator.MakeServerWithMiddleware(dbround, genesisFile, configFile, addr, blockMiddleware) + server, generator := generator.MakeServerWithMiddleware(dbround, genesisFile, configFile, verbose, addr, blockMiddleware) // Start the server go func() { diff --git a/tools/block-generator/runner/runner.go b/tools/block-generator/runner/runner.go index 8205e7427a..1bb3cd9cf6 100644 --- a/tools/block-generator/runner/runner.go +++ b/tools/block-generator/runner/runner.go @@ -49,7 +49,8 @@ func init() { RunnerCmd.Flags().StringVarP(&runnerArgs.PostgresConnectionString, "postgres-connection-string", "c", "", "Postgres connection string.") RunnerCmd.Flags().DurationVarP(&runnerArgs.RunDuration, "test-duration", "d", 5*time.Minute, "Duration to use for each scenario.") RunnerCmd.Flags().StringVarP(&runnerArgs.ReportDirectory, "report-directory", "r", "", "Location to place test reports.") - RunnerCmd.Flags().StringVarP(&runnerArgs.LogLevel, "log-level", "l", "error", "LogLevel to use when starting Conduit. [panic, fatal, error, warn, info, debug, trace]") + RunnerCmd.Flags().BoolVarP(&runnerArgs.RunnerVerbose, "verbose", "v", false, "If set the runner will print debugging information from the generator and ledger.") + RunnerCmd.Flags().StringVarP(&runnerArgs.ConduitLogLevel, "conduit-log-level", "l", "error", "LogLevel to use when starting Conduit. [panic, fatal, error, warn, info, debug, trace]") RunnerCmd.Flags().StringVarP(&runnerArgs.CPUProfilePath, "cpuprofile", "", "", "Path where Conduit writes its CPU profile.") RunnerCmd.Flags().BoolVarP(&runnerArgs.ResetReportDir, "reset-report-dir", "", false, "If set any existing report directory will be deleted before running tests.") RunnerCmd.Flags().BoolVarP(&runnerArgs.RunValidation, "validate", "", false, "If set the validator will run after test-duration has elapsed to verify data is correct. An extra line in each report indicates validator success or failure.") diff --git a/tools/block-generator/scenarios/config.allmixed.jumbo.yml b/tools/block-generator/scenarios/config.allmixed.jumbo.yml new file mode 100644 index 0000000000..bc6f1a56a6 --- /dev/null +++ b/tools/block-generator/scenarios/config.allmixed.jumbo.yml @@ -0,0 +1,31 @@ +name: "App Create" +tx_per_block: 25000 + +# transaction distribution +tx_pay_fraction: 0.25 +tx_asset_fraction: 0.25 +tx_app_fraction: 0.50 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.1 +asset_close_fraction: 0.05 +asset_xfer_fraction: 0.849 +asset_delete_fraction: 0 + +# app kind config +app_boxes_fraction: 0.5 +app_swap_fraction: 0.5 + +# app boxes config +app_boxes_create_fraction: 0.01 +app_boxes_optin_fraction: 0.1 +app_boxes_call_fraction: 0.89 + +# app swap config +app_swap_create_fraction: 1.0 + diff --git a/tools/block-generator/scenarios/config.allmixed.small.yml b/tools/block-generator/scenarios/config.allmixed.small.yml new file mode 100644 index 0000000000..f4ef216577 --- /dev/null +++ b/tools/block-generator/scenarios/config.allmixed.small.yml @@ -0,0 +1,31 @@ +name: "App Create" +tx_per_block: 100 + +# transaction distribution +tx_pay_fraction: 0.25 +tx_asset_fraction: 0.25 +tx_app_fraction: 0.50 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +asset_create_fraction: 0.001 +asset_optin_fraction: 0.1 +asset_close_fraction: 0.05 +asset_xfer_fraction: 0.849 +asset_delete_fraction: 0 + +# app kind config +app_boxes_fraction: 0.5 +app_swap_fraction: 0.5 + +# app boxes config +app_boxes_create_fraction: 0.01 +app_boxes_optin_fraction: 0.1 +app_boxes_call_fraction: 0.89 + +# app swap config +app_swap_create_fraction: 1.0 + diff --git a/tools/block-generator/scenarios/config.appboxes.small.yml b/tools/block-generator/scenarios/config.appboxes.small.yml new file mode 100644 index 0000000000..f8fd7e4e6a --- /dev/null +++ b/tools/block-generator/scenarios/config.appboxes.small.yml @@ -0,0 +1,23 @@ +name: "App Create" +tx_per_block: 100 + +# transaction distribution +# tx_pay_fraction: 0.01 +# tx_asset_fraction: 0.01 +# tx_app_fraction: 0.98 +tx_app_fraction: 1 + +# payment config +pay_acct_create_fraction: 0.02 +pay_xfer_fraction: 0.98 + +# asset config +tx_asset_create_fraction: 1.0 + +# app kind config +app_boxes_fraction: 1.0 + +# app boxes config +app_boxes_create_fraction: 0.01 +app_boxes_optin_fraction: 0.1 +app_boxes_call_fraction: 0.89 diff --git a/tools/block-generator/scenarios/config.app.create.yml b/tools/block-generator/scenarios/config.appcreate.small.yml similarity index 85% rename from tools/block-generator/scenarios/config.app.create.yml rename to tools/block-generator/scenarios/config.appcreate.small.yml index 55e9db7ecb..3e02ae4318 100644 --- a/tools/block-generator/scenarios/config.app.create.yml +++ b/tools/block-generator/scenarios/config.appcreate.small.yml @@ -2,8 +2,8 @@ name: "App Create" tx_per_block: 100 # transaction distribution -tx_asset_fraction: 0.5 -tx_app_fraction: 0.5 +tx_asset_fraction: 0.05 +tx_app_fraction: 0.95 # asset config tx_asset_create_fraction: 1.0