From 8f676129aecbf135e4a7a0b843a9e442e24a66d6 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Fri, 25 Oct 2024 07:55:01 -0400 Subject: [PATCH 1/5] Adds an optimism specials.csv file --- src/other/install/init_configs.cmake | 27 ++++++++++--------- .../install/per-chain/optimism/specials.csv | 3 +++ 2 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 src/other/install/per-chain/optimism/specials.csv diff --git a/src/other/install/init_configs.cmake b/src/other/install/init_configs.cmake index 68c3f644ed..990aac7d5c 100644 --- a/src/other/install/init_configs.cmake +++ b/src/other/install/init_configs.cmake @@ -89,23 +89,24 @@ PrintLine("Copying files and folder to config...") # --------------------------------------------------------------- # Function InFolder InFile OutFolder -CopyNotPresent (${INSTALL_SOURCE} "trueBlocks.toml" ${INSTALL_DEST}) +CopyNotPresent (${INSTALL_SOURCE} "trueBlocks.toml" ${INSTALL_DEST}) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "allocs.csv" ${INSTALL_DEST}/config/mainnet/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis "allocs.csv" ${INSTALL_DEST}/config/gnosis/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "allocs.csv" ${INSTALL_DEST}/config/sepolia/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/goerli "allocs.csv" ${INSTALL_DEST}/config/goerli/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/polygon "allocs.csv" ${INSTALL_DEST}/config/polygon/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "allocs.csv" ${INSTALL_DEST}/config/mainnet/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis "allocs.csv" ${INSTALL_DEST}/config/gnosis/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "allocs.csv" ${INSTALL_DEST}/config/sepolia/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/goerli "allocs.csv" ${INSTALL_DEST}/config/goerli/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/polygon "allocs.csv" ${INSTALL_DEST}/config/polygon/) -CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/mainnet/) -CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/gnosis/) -CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/sepolia/) +CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/mainnet/) +CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/gnosis/) +CopyAndTouchIgnorePresent (${INSTALL_SOURCE}/names/ "names.tab" ${INSTALL_DEST}/config/sepolia/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "specials.csv" ${INSTALL_DEST}/config/mainnet/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis "specials.csv" ${INSTALL_DEST}/config/gnosis/) -CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "specials.csv" ${INSTALL_DEST}/config/sepolia/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/mainnet "specials.csv" ${INSTALL_DEST}/config/mainnet/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/gnosis "specials.csv" ${INSTALL_DEST}/config/gnosis/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/sepolia "specials.csv" ${INSTALL_DEST}/config/sepolia/) +CopyIgnorePresent (${INSTALL_SOURCE}/per-chain/optimism "specials.csv" ${INSTALL_DEST}/config/optimism/) -CopyNotPresent (${INSTALL_SOURCE}/names/ "names_custom.tab" ${INSTALL_DEST}/config/mainnet/) +CopyNotPresent (${INSTALL_SOURCE}/names/ "names_custom.tab" ${INSTALL_DEST}/config/mainnet/) CopyFolder (${INSTALL_SOURCE}/abis/known-000/ ${INSTALL_DEST}/abis/known-000/) CopyFolder (${INSTALL_SOURCE}/abis/known-005/ ${INSTALL_DEST}/abis/known-005/) diff --git a/src/other/install/per-chain/optimism/specials.csv b/src/other/install/per-chain/optimism/specials.csv new file mode 100644 index 0000000000..6b3826eea8 --- /dev/null +++ b/src/other/install/per-chain/optimism/specials.csv @@ -0,0 +1,3 @@ +component,block/epoch,name,timestamp,date,description +execution,0,first,1636665386,2021-11-11T21:16:26,The chain's first block +execution,105235063,Bedrock,1686068903,2023-06-06T16:28:23,The Bedrock hard fork block From 21c6274b8c07ee074c3f96816451810594905abe Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Fri, 25 Oct 2024 07:56:52 -0400 Subject: [PATCH 2/5] Small improvements to the scraper --- .../internal/names/handle_autoname_integration_test.go | 2 +- src/apps/chifra/internal/scrape/scrape_consolidate.go | 2 +- src/apps/chifra/internal/scrape/scrape_prepare.go | 2 +- src/apps/chifra/pkg/index/chunk_write.go | 5 ++--- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/apps/chifra/internal/names/handle_autoname_integration_test.go b/src/apps/chifra/internal/names/handle_autoname_integration_test.go index b5a24d0d7f..79a3af2956 100644 --- a/src/apps/chifra/internal/names/handle_autoname_integration_test.go +++ b/src/apps/chifra/internal/names/handle_autoname_integration_test.go @@ -31,7 +31,7 @@ func TestNamesOptions_autoname(t *testing.T) { if err != nil { t.Errorf("Error making backup = %+v", err) } - defer backup.Restore() // put the file back + defer backup.Restore() // put the file back if it still exists (i.e., an error occurred) type fields struct { Autoname string diff --git a/src/apps/chifra/internal/scrape/scrape_consolidate.go b/src/apps/chifra/internal/scrape/scrape_consolidate.go index e64a18e5f8..eaf178fc38 100644 --- a/src/apps/chifra/internal/scrape/scrape_consolidate.go +++ b/src/apps/chifra/internal/scrape/scrape_consolidate.go @@ -106,7 +106,7 @@ func (bm *BlazeManager) Consolidate(ctx context.Context, blocks []base.Blknum) e } else { report.Snapped = isSnap report.FileSize = file.FileSize(chunkPath) - report.Report() + logger.Info(report.Report()) } if err = bm.opts.NotifyChunkWritten(chunk, chunkPath); err != nil { return err diff --git a/src/apps/chifra/internal/scrape/scrape_prepare.go b/src/apps/chifra/internal/scrape/scrape_prepare.go index aec5709ac8..c5c4d9b418 100644 --- a/src/apps/chifra/internal/scrape/scrape_prepare.go +++ b/src/apps/chifra/internal/scrape/scrape_prepare.go @@ -65,7 +65,7 @@ func (opts *ScrapeOptions) Prepare() (ok bool, err error) { } else { report.Snapped = true // assumes block zero is a snap to grid (which it is in a sense) report.FileSize = file.FileSize(indexPath) - report.Report() + logger.Info(report.Report()) } if err = opts.NotifyChunkWritten(chunk, indexPath); err != nil { return false, err diff --git a/src/apps/chifra/pkg/index/chunk_write.go b/src/apps/chifra/pkg/index/chunk_write.go index 10490ca600..2cd40d5900 100644 --- a/src/apps/chifra/pkg/index/chunk_write.go +++ b/src/apps/chifra/pkg/index/chunk_write.go @@ -12,7 +12,6 @@ import ( "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file" - "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types" ) @@ -24,13 +23,13 @@ type writeReport struct { Snapped bool } -func (c *writeReport) Report() { +func (c *writeReport) Report() string { report := `Wrote {%d} address and {%d} appearance records to {$INDEX/%s.bin}` if c.Snapped { report += ` @(snapped to grid)}` } report += " (size: {%d} , span: {%d})" - logger.Info(colors.ColoredWith(fmt.Sprintf(report, c.nAddresses, c.nAppearances, c.Range, c.FileSize, c.Range.Span()), colors.BrightBlue)) + return colors.ColoredWith(fmt.Sprintf(report, c.nAddresses, c.nAppearances, c.Range, c.FileSize, c.Range.Span()), colors.BrightBlue) } func (chunk *Chunk) Write(chain string, publisher base.Address, fileName string, addrAppearanceMap map[string][]types.AppRecord, nApps int) (*writeReport, error) { From 1c31a6c0beeb3ab8903eb504eea00d9efe10fbe9 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Fri, 25 Oct 2024 07:58:22 -0400 Subject: [PATCH 3/5] Preparing to implement chifra scrape --touch --- .../chifra/internal/scrape/save_timestamps.go | 2 +- src/apps/chifra/internal/scrape/validate.go | 4 ++ src/apps/chifra/pkg/base/baddresses.go | 1 + src/apps/chifra/pkg/manifest/remove_chunk.go | 58 +++++++++++-------- src/apps/chifra/pkg/rpc/is_node.go | 6 ++ 5 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/apps/chifra/internal/scrape/save_timestamps.go b/src/apps/chifra/internal/scrape/save_timestamps.go index 53bb6eb3ed..24a382ccc9 100644 --- a/src/apps/chifra/internal/scrape/save_timestamps.go +++ b/src/apps/chifra/internal/scrape/save_timestamps.go @@ -42,7 +42,7 @@ func (bm *BlazeManager) WriteTimestamps(ctx context.Context, blocks []base.Blknu // we need to catch up (for example, the user truncated the timestamps file while debugging) // don't get more than maxBlocks at a time cnt := 0 - maxBlocks := 1000 + maxBlocks := 2000 for block := nTimestamps; block < blocks[0] && cnt < maxBlocks; block++ { if ctx.Err() != nil { // This means the context got cancelled, i.e. we got a SIGINT. diff --git a/src/apps/chifra/internal/scrape/validate.go b/src/apps/chifra/internal/scrape/validate.go index b3cde11f5d..64de1f5f6c 100644 --- a/src/apps/chifra/internal/scrape/validate.go +++ b/src/apps/chifra/internal/scrape/validate.go @@ -81,6 +81,10 @@ func (opts *ScrapeOptions) validateScrape() error { fmt.Println(validate.Usage("The index ({0}) is ahead of the chain ({1}).", fmt.Sprintf("%d", m), fmt.Sprintf("%d", meta.Latest))) } + if opts.Touch > 0 && opts.Touch < meta.Finalized { + return validate.Usage("The index is already finalized ({0}) past the touch block ({1}).", fmt.Sprintf("%d", meta.Finalized), fmt.Sprintf("%d", opts.Touch)) + } + if len(opts.Publisher) > 0 { err := validate.ValidateExactlyOneAddr([]string{opts.Publisher}) if err != nil { diff --git a/src/apps/chifra/pkg/base/baddresses.go b/src/apps/chifra/pkg/base/baddresses.go index efe955b196..fe93880674 100644 --- a/src/apps/chifra/pkg/base/baddresses.go +++ b/src/apps/chifra/pkg/base/baddresses.go @@ -8,4 +8,5 @@ var ( BlockRewardSender = HexToAddress("0x0000000000000000000000000000004d696e6572") // The word "Miner" in hex UncleRewardSender = HexToAddress("0x000000000000000000000000000000556e636c65") // The word "Uncle" in hex WithdrawalSender = HexToAddress("0x000000000000000000005769746864726177616C") // The word "Withdrawal" in hex + SkippedSender = HexToAddress("0x00000000000000000000000000536B6970706564") // The word "Skipped" in hex ) diff --git a/src/apps/chifra/pkg/manifest/remove_chunk.go b/src/apps/chifra/pkg/manifest/remove_chunk.go index 2b21430f6c..54583531e1 100644 --- a/src/apps/chifra/pkg/manifest/remove_chunk.go +++ b/src/apps/chifra/pkg/manifest/remove_chunk.go @@ -38,14 +38,19 @@ func RemoveChunk(chain string, publisher base.Address, bloomFn, indexFn string) } }() - var man *Manifest - man, err = LoadManifest(chain, publisher, LocalCache) - if err != nil { - return err - } + exists := file.FileExists(manifestFn) - if _, err = file.Copy(manifestBackup, manifestFn); err != nil { - return err + var man *Manifest + if exists { + man, err = LoadManifest(chain, publisher, LocalCache) + if err != nil { + return err + } + if _, err = file.Copy(manifestBackup, manifestFn); err != nil { + return err + } + } else { + man = &Manifest{} } if _, err = file.Copy(indexBackup, indexFn); err != nil { @@ -64,28 +69,31 @@ func RemoveChunk(chain string, publisher base.Address, bloomFn, indexFn string) return err } - removedRange, err1 := base.RangeFromFilenameE(bloomFn) - if err1 != nil { - err = err1 - return err - } + if exists { + removedRange, err1 := base.RangeFromFilenameE(bloomFn) + if err1 != nil { + err = err1 + return err + } - newChunks := []types.ChunkRecord{} - for _, chunk := range man.Chunks { - chunkRange := base.RangeFromRangeString(chunk.Range) - if chunkRange.EarlierThan(removedRange) { - newChunks = append(newChunks, chunk) - // fmt.Println(colors.Green, "Keeping", chunk.Range, colors.Off) - // } else { - // fmt.Println(colors.Red, "Removing", chunk.Range, colors.Off) + newChunks := []types.ChunkRecord{} + for _, chunk := range man.Chunks { + chunkRange := base.RangeFromRangeString(chunk.Range) + if chunkRange.EarlierThan(removedRange) { + newChunks = append(newChunks, chunk) + // fmt.Println(colors.Green, "Keeping", chunk.Range, colors.Off) + // } else { + // fmt.Println(colors.Red, "Removing", chunk.Range, colors.Off) + } } - } - man.Chunks = newChunks - if err = man.SaveManifest(chain, config.PathToManifest(chain)); err != nil { - return err + man.Chunks = newChunks + if err = man.SaveManifest(chain, config.PathToManifest(chain)); err != nil { + return err + } + + os.Remove(manifestBackup) } - os.Remove(manifestBackup) os.Remove(indexBackup) os.Remove(bloomBackup) diff --git a/src/apps/chifra/pkg/rpc/is_node.go b/src/apps/chifra/pkg/rpc/is_node.go index efb0c2a5cc..c2b574b00c 100644 --- a/src/apps/chifra/pkg/rpc/is_node.go +++ b/src/apps/chifra/pkg/rpc/is_node.go @@ -7,6 +7,8 @@ import ( "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/prefunds" ) @@ -21,6 +23,10 @@ import ( // false, when in fact the node may be an archive node. func (conn *Connection) IsNodeArchive() bool { thePath := filepath.Join(config.MustGetPathToChainConfig(conn.Chain), "allocs.csv") + if !file.FileExists(thePath) { + logger.Warn("No pre-allocation file found at", thePath, "assuming an archive node") + return true + } largest, err := prefunds.GetLargestPrefund(conn.Chain, thePath) if err != nil { return false From 5a57f318f4b87f8cc05c62db0463ee3f558ce851 Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Fri, 25 Oct 2024 07:59:10 -0400 Subject: [PATCH 4/5] Implements chifra scrape --touch --- .../chifra/internal/scrape/handle_touch.go | 100 +++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/src/apps/chifra/internal/scrape/handle_touch.go b/src/apps/chifra/internal/scrape/handle_touch.go index 18467545c4..c1a3362445 100644 --- a/src/apps/chifra/internal/scrape/handle_touch.go +++ b/src/apps/chifra/internal/scrape/handle_touch.go @@ -5,15 +5,113 @@ package scrapePkg import ( + "context" + "encoding/binary" "fmt" + "os" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/sigintTrap" + "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib" "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types" ) func (opts *ScrapeOptions) HandleTouch(rCtx *output.RenderCtx) error { + chain := opts.Globals.Chain fetchData := func(modelChan chan types.Modeler, errorChan chan error) { - errorChan <- fmt.Errorf("chifra scrape --touch is not yet implemented") + meta, _ := opts.Conn.GetMetaData(false) + rng := base.FileRange{ + First: meta.Finalized, + Last: opts.Touch, + } + indexPath := index.ToIndexPath(rng.RangeToFilename(chain)) + stagePath := index.ToStagingPath(rng.RangeToFilename(chain)) + var chunk index.Chunk + var bm = BlazeManager{} + file.StringToAsciiFile(stagePath, base.SkippedSender.Hex()+"\t0\t0\n") + appMap, _, nAppearances := bm.AsciiFileToAppearanceMap(stagePath) + if report, err := chunk.Write(chain, base.ZeroAddr, indexPath, appMap, nAppearances); err != nil { + errorChan <- err + } else { + modelChan <- &types.Message{Msg: report.Report()} + } + if reportStr, err := opts.backfillTimestamps(); err != nil { + errorChan <- err + } else { + modelChan <- &types.Message{Msg: reportStr} + } } return output.StreamMany(rCtx, fetchData, opts.Globals.OutputOpts()) } + +func (opts *ScrapeOptions) backfillTimestamps() (string, error) { + chain := opts.Globals.Chain + tsPath := config.PathToTimestamps(chain) + logger.Info(tsPath) + + sigintCtx, cancel := context.WithCancel(context.Background()) + cleanOnQuit := func() { + // We only print a warning here, as the scrape.pid file will be + // removed by the deferred function + logger.Warn(sigintTrap.TrapMessage) + } + trapChannel := sigintTrap.Enable(sigintCtx, cancel, cleanOnQuit) + defer sigintTrap.Disable(trapChannel) + + tmpPath := tsPath + ".backup" + backup, err := file.MakeBackup(tmpPath, tsPath) + if err != nil { + return "", fmt.Errorf("error making backup = %+v", err) + } + defer backup.Restore() // put the file back if it still exists (i.e., an error occurred) + + file.Remove(tsPath) + fp, err := os.OpenFile(tsPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return "", err + } + defer func() { + tslib.ClearCache(chain) + fp.Close() + }() + + if nTimestamps, err := tslib.NTimestamps(chain); err != nil { + return "", err + } else { + logger.Info("nTimestamps:", nTimestamps) + if nTimestamps > opts.Touch { + return "", fmt.Errorf("nTimestamps > opts.Touch - should not happen") + } + + firstTs := uint32(0) + for block := uint32(0); block <= uint32(opts.Touch); block++ { + if sigintCtx.Err() != nil { + // This means the context got cancelled, i.e. we got a SIGINT. + // The backup will be restored by the deferred function + break + } + if block == 0 { + firstTs = uint32(opts.Conn.GetBlockTimestamp(base.Value(block))) + } + ts := tslib.TimestampRecord{ + Bn: block, + Ts: uint32(block + firstTs), + } + msg := fmt.Sprintf("Backfilling timestamps at block %d", block) + logger.Progress(true, msg) + if err := binary.Write(fp, binary.LittleEndian, &ts); err != nil { + return "", err + } + } + msg := fmt.Sprintf("{%s} were backfilled to block {%d}", "Timestamps", opts.Touch) + ret := colors.ColoredWith(msg, colors.BrightBlue) + os.Remove(tmpPath) + return ret, nil + } +} From 3d2a3593d0a0bf33fd1fa747d3d20360d19ae31a Mon Sep 17 00:00:00 2001 From: Thomas Jay Rush Date: Fri, 25 Oct 2024 08:22:39 -0400 Subject: [PATCH 5/5] Fixes linter --- src/apps/chifra/internal/scrape/handle_touch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/chifra/internal/scrape/handle_touch.go b/src/apps/chifra/internal/scrape/handle_touch.go index c1a3362445..a5f973d721 100644 --- a/src/apps/chifra/internal/scrape/handle_touch.go +++ b/src/apps/chifra/internal/scrape/handle_touch.go @@ -34,7 +34,7 @@ func (opts *ScrapeOptions) HandleTouch(rCtx *output.RenderCtx) error { stagePath := index.ToStagingPath(rng.RangeToFilename(chain)) var chunk index.Chunk var bm = BlazeManager{} - file.StringToAsciiFile(stagePath, base.SkippedSender.Hex()+"\t0\t0\n") + _ = file.StringToAsciiFile(stagePath, base.SkippedSender.Hex()+"\t0\t0\n") appMap, _, nAppearances := bm.AsciiFileToAppearanceMap(stagePath) if report, err := chunk.Write(chain, base.ZeroAddr, indexPath, appMap, nAppearances); err != nil { errorChan <- err