From 3ea3702a1edbea32116090310123eda54112f99d Mon Sep 17 00:00:00 2001 From: Willian Mitsuda Date: Tue, 3 Dec 2024 18:26:16 -0300 Subject: [PATCH] --wip-- [skip ci] --- erigon-lib/state/aggregator.go | 1 + erigon-lib/state/history.go | 2 ++ erigon-lib/state/inverted_index.go | 1 + erigon-lib/state/merge.go | 2 ++ 4 files changed, 6 insertions(+) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 7cc16566ed2..b1f7766e1f4 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -609,6 +609,7 @@ func (sf AggV3StaticFiles) CleanupOnError() { } } +// TODO: implement multiencseq support for collation func (a *Aggregator) buildFiles(ctx context.Context, step uint64) error { a.logger.Debug("[agg] collate and build", "step", step, "collate_workers", a.collateAndBuildWorkers, "merge_workers", a.mergeWorkers, "compress_workers", a.d[kv.AccountsDomain].compressCfg.Workers) diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 78ddeb764c2..3e0d3413d04 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -379,6 +379,7 @@ func (h *History) buildVI(ctx context.Context, historyIdxPath string, hist, efHi histReader := seg.NewReader(hist.MakeGetter(), h.compression) efHistReader := seg.NewReader(efHist.MakeGetter(), h.InvertedIndex.compression) + // TODO: dedup for { histReader.Reset(0) efHistReader.Reset(0) @@ -572,6 +573,7 @@ func (c HistoryCollation) Close() { } } +// TODO: dedup collate code for ii // [txFrom; txTo) func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv.Tx) (HistoryCollation, error) { if h.snapshotsDisabled { diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index f0fc15c35ec..398d5133ecd 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -1015,6 +1015,7 @@ func (iit *InvertedIndexRoTx) IterateChangedKeys(startTxNum, endTxNum uint64, ro return ii1 } +// TODO: add multiencseq support to collate // collate [stepFrom, stepTo) func (ii *InvertedIndex) collate(ctx context.Context, step uint64, roTx kv.Tx) (InvertedIndexCollation, error) { stepTo := step + 1 diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go index 3f3f40476d5..4769bce0b8a 100644 --- a/erigon-lib/state/merge.go +++ b/erigon-lib/state/merge.go @@ -338,6 +338,7 @@ func (ht *HistoryRoTx) staticFilesInRange(r HistoryRanges) (indexFiles, historyF return } +// TODO: add support to multiencseq func mergeEfs(preval, val, buf []byte) ([]byte, error) { preef, _ := eliasfano32.ReadEliasFano(preval) ef, _ := eliasfano32.ReadEliasFano(val) @@ -778,6 +779,7 @@ func (ht *HistoryRoTx) mergeFiles(ctx context.Context, indexFiles, historyFiles }) } } + // TODO: dedup // In the loop below, the pair `keyBuf=>valBuf` is always 1 item behind `lastKey=>lastVal`. // `lastKey` and `lastVal` are taken from the top of the multi-way merge (assisted by the CursorHeap cp), but not processed right away // instead, the pair from the previous iteration is processed first - `keyBuf=>valBuf`. After that, `keyBuf` and `valBuf` are assigned