Skip to content

Commit 460b04c

Browse files
authored
Early Stages of EventLog processing (#1)
Initial port of Ujo event log processing objects
1 parent d1a5446 commit 460b04c

34 files changed

+783
-89
lines changed
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Numerics;
6+
using System.Text;
7+
using System.Threading.Tasks;
8+
using Castle.Components.DictionaryAdapter;
9+
using Nethereum.ABI.FunctionEncoding.Attributes;
10+
using Nethereum.BlockchainProcessing.Handlers;
11+
using Nethereum.BlockchainProcessing.Processing;
12+
using Nethereum.BlockchainProcessing.Processing.Logs;
13+
using Nethereum.BlockchainProcessing.Web3Abstractions;
14+
using Nethereum.Contracts;
15+
using Nethereum.Contracts.Extensions;
16+
using Nethereum.RPC.Eth.DTOs;
17+
using Xunit;
18+
19+
namespace Nethereum.BlockchainProcessing.Samples
20+
{
21+
public class EventLogEnumeration
22+
{
23+
/*
24+
Solidity Contract Excerpt
25+
* event Transfer(address indexed _from, address indexed _to, uint256 indexed _value);
26+
Other contracts may have transfer events with different signatures, this won't work for those.
27+
*/
28+
[Event("Transfer")]
29+
public class TransferEvent
30+
{
31+
[Parameter("address", "_from", 1, true)]
32+
public string From {get; set;}
33+
34+
[Parameter("address", "_to", 2, true)]
35+
public string To {get; set;}
36+
37+
[Parameter("uint256", "_value", 3, true)]
38+
public BigInteger Value {get; set;}
39+
}
40+
41+
public class TransferEventProcessor : ILogProcessor
42+
{
43+
public List<(FilterLog, EventLog<TransferEvent>)> ProcessedEvents = new List<(FilterLog, EventLog<TransferEvent>)>();
44+
public List<(FilterLog, Exception)> DecodingErrors = new List<(FilterLog, Exception)>();
45+
46+
public bool IsLogForEvent(FilterLog log)
47+
{
48+
return log.IsLogForEvent<TransferEvent>();
49+
}
50+
51+
public Task ProcessLogsAsync(params FilterLog[] eventLogs)
52+
{
53+
foreach (var eventLog in eventLogs)
54+
{
55+
try
56+
{
57+
var eventDto = eventLog.DecodeEvent<TransferEvent>();
58+
ProcessedEvents.Add((eventLog, eventDto));
59+
60+
}
61+
catch (Exception ex)
62+
{
63+
DecodingErrors.Add((eventLog, ex));
64+
}
65+
}
66+
67+
return Task.CompletedTask;
68+
}
69+
}
70+
71+
public class CatchAllEventProcessor : ILogProcessor
72+
{
73+
public List<FilterLog> ProcessedEvents = new List<FilterLog>();
74+
75+
public bool IsLogForEvent(FilterLog log)
76+
{
77+
return true;
78+
}
79+
80+
public Task ProcessLogsAsync(params FilterLog[] eventLogs)
81+
{
82+
ProcessedEvents.AddRange(eventLogs);
83+
return Task.CompletedTask;
84+
}
85+
}
86+
87+
[Fact]
88+
public async Task RunOnce()
89+
{
90+
var web3Wrapper = new Web3Wrapper("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60");
91+
92+
var transferEventProcessor = new TransferEventProcessor();
93+
var catchAllEventProcessor = new CatchAllEventProcessor();
94+
var eventProcessors = new ILogProcessor[] {catchAllEventProcessor, transferEventProcessor};
95+
96+
var logProcessor = new BlockchainLogProcessor(web3Wrapper, eventProcessors);
97+
98+
var progressFileNameAndPath = Path.Combine(Path.GetTempPath(), "BlockProcess.json");
99+
if(File.Exists(progressFileNameAndPath)) File.Delete(progressFileNameAndPath);
100+
101+
var progressRepository = new JsonBlockProcessProgressRepository(progressFileNameAndPath);
102+
var progressService = new PreDefinedRangeBlockchainProcessingProgressService(
103+
3146684, 3146684, progressRepository);
104+
105+
var batchProcessorService = new BlockchainBatchProcessorService(
106+
logProcessor, progressService, maxNumberOfBlocksPerBatch: 1);
107+
108+
await batchProcessorService.ProcessLatestBlocks();
109+
110+
Assert.Single(transferEventProcessor.ProcessedEvents);
111+
Assert.Equal(7, catchAllEventProcessor.ProcessedEvents.Count);
112+
113+
Assert.Equal((ulong?)3146684, await progressRepository.GetLatestAsync());
114+
115+
}
116+
117+
[Fact]
118+
public async Task RunContinually()
119+
{
120+
const ulong StartingBlockNumber = 3146684;
121+
var web3Wrapper = new Web3Wrapper("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60");
122+
123+
var transferEventProcessor = new TransferEventProcessor();
124+
var catchAllEventProcessor = new CatchAllEventProcessor();
125+
var eventProcessors = new ILogProcessor[] {catchAllEventProcessor, transferEventProcessor};
126+
127+
var logProcessor = new BlockchainLogProcessor(web3Wrapper, eventProcessors);
128+
129+
var progressFileNameAndPath = Path.Combine(Path.GetTempPath(), "BlockProcess.json");
130+
if(File.Exists(progressFileNameAndPath)) File.Delete(progressFileNameAndPath);
131+
132+
var progressRepository = new JsonBlockProcessProgressRepository(progressFileNameAndPath);
133+
134+
//this will get the last block on the chain each time a "to" block is requested
135+
var progressService = new LatestBlockBlockchainProcessingProgressService(
136+
web3Wrapper, StartingBlockNumber, progressRepository);
137+
138+
var batchProcessorService = new BlockchainBatchProcessorService(
139+
logProcessor, progressService, maxNumberOfBlocksPerBatch: 10);
140+
141+
var iterations = 0;
142+
143+
//iterate until we reach an arbitrary ending block
144+
//to process continually - remove the condition from the while loop
145+
while (progressRepository.Latest < (StartingBlockNumber + 100))
146+
{
147+
await batchProcessorService.ProcessLatestBlocks();
148+
iterations++;
149+
}
150+
151+
Assert.Equal(10, iterations);
152+
Assert.Equal(1533, catchAllEventProcessor.ProcessedEvents.Count);
153+
Assert.Equal(40, transferEventProcessor.ProcessedEvents.Count);
154+
155+
//events on other contracts may have same name and input parameter types
156+
//however they may differ in the number of indexed fields
157+
//this leads to decoding errors
158+
//it's not a problem - just something to be aware of
159+
Assert.Equal(201, transferEventProcessor.DecodingErrors.Count);
160+
}
161+
}
162+
}

Nethereum.BlockchainProcessing.Samples/README.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,16 @@ Here are some quick start samples to demonstrate processing the chain.
44
They are written as unit tests against known data on a publicly available testnet.
55
This means you can run them and expect the same results.
66

7+
## Block and Transaction Processing (includes Event Logs)
8+
If you need transactions and logs then use these examples.
9+
710
* [Block And Transaction Enumeration](BlockAndTransactionEnumeration.cs)
811
* [Filtering Transactions](FilterTransactions.cs)
912
* [Listening For A Specific Event](ListeningForASpecificEvent.cs)
1013
* [Listening For A Specific Function Call](ListeningForASpecificFunctionCall.cs)
1114
* [Conditional Transaction Routing](ConditionalTransactionRouting.cs)
12-
* [Conditional Transaction Log Routing](ConditionalTransactionLogRouting.cs)
15+
* [Conditional Transaction Log Routing](ConditionalTransactionLogRouting.cs)
16+
17+
## Event Log Processing (excludes transactions)
18+
If you are ONLY interested in event logs - then this is a much faster way of enumerating the chain and listening for events.
19+
* [EventLogEnumeration](EventLogEnumeration.cs)

Nethereum.BlockchainProcessing.Tests/Processing/BlockchainProcessorTests.cs

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using Moq;
1+
using System;
2+
using Moq;
23
using Nethereum.BlockchainProcessing.Processing;
34
using Nethereum.JsonRpc.Client;
45
using System.Threading;
@@ -42,7 +43,7 @@ public async Task
4243
await Processor.ExecuteAsync(startBlock: 0, endBlock: 0, cancellationToken: cancellationTokenSource.Token);
4344

4445
MockProcessingStrategy.VerifyAll();
45-
MockProcessingStrategy.Verify(p => p.ProcessBlockAsync(It.IsAny<long>()), Times.Never);
46+
MockProcessingStrategy.Verify(p => p.ProcessBlockAsync(It.IsAny<ulong>()), Times.Never);
4647

4748
}
4849

@@ -52,8 +53,8 @@ public class When_Block_Range_Is_Specified : BlockchainProcessorTests
5253
public async Task
5354
Will_Process_The_Requested_Block_Range()
5455
{
55-
const long StartBlock = 1000;
56-
const long EndBlock = 1205;
56+
const ulong StartBlock = 1000;
57+
const ulong EndBlock = 1205;
5758

5859
for (var block = StartBlock; block <= EndBlock; block++)
5960
{
@@ -68,9 +69,9 @@ public async Task
6869
[Fact]
6970
public async Task Will_Wait_For_Minimum_Block_Confirmations()
7071
{
71-
const int RequiredBlockConfirmations = 6;
72-
long maxBlockNumber = 0;
73-
int numberOfWaitCycles = 0;
72+
const uint RequiredBlockConfirmations = 6;
73+
ulong maxBlockNumber = 0;
74+
uint numberOfWaitCycles = 0;
7475
int blocksProcessed = 0;
7576

7677
MockProcessingStrategy
@@ -83,8 +84,8 @@ public async Task Will_Wait_For_Minimum_Block_Confirmations()
8384
.ReturnsAsync(() => maxBlockNumber);
8485

8586
MockProcessingStrategy
86-
.Setup(s => s.WaitForNextBlock(It.IsAny<int>()))
87-
.Callback<int>((retryNumber) => numberOfWaitCycles++)
87+
.Setup(s => s.WaitForNextBlock(It.IsAny<uint>()))
88+
.Callback<uint>((retryNumber) => numberOfWaitCycles++)
8889
.Returns(Task.CompletedTask);
8990

9091
MockProcessingStrategy
@@ -124,10 +125,10 @@ public async Task Will_Retry_Up_To_Retry_Limit_And_Pause_Between_Each_Attempt()
124125

125126
MockProcessingStrategy
126127
.Setup(p => p.ProcessBlockAsync(0))
127-
.Callback<long>((blkNum) => timesThrown++)
128+
.Callback<ulong>((blkNum) => timesThrown++)
128129
.Throws(blockProcessingException);
129130

130-
for (var retryNum = 0; retryNum < MaxRetries; retryNum++)
131+
for (uint retryNum = 0; retryNum < MaxRetries; retryNum++)
131132
{
132133
MockProcessingStrategy
133134
.Setup(s => s.PauseFollowingAnError(retryNum))
@@ -157,7 +158,7 @@ public async Task
157158

158159
MockProcessingStrategy
159160
.Setup(p => p.ProcessBlockAsync(1))
160-
.Callback<long>(blkNum => cancellationTokenSource.Cancel())
161+
.Callback<ulong>(blkNum => cancellationTokenSource.Cancel())
161162
.Returns(Task.CompletedTask)
162163
.Verifiable("ProcessBlockAsync should have been called for Block 1");
163164

@@ -172,10 +173,10 @@ public async Task
172173
[Fact]
173174
public async Task Will_Wait_For_Minimum_Block_Confirmations()
174175
{
175-
const int RequiredBlockConfirmations = 6;
176-
long maxBlockNumber = 0;
177-
int numberOfWaitCycles = 0;
178-
int blocksProcessed = 0;
176+
const uint RequiredBlockConfirmations = 6;
177+
ulong maxBlockNumber = 0;
178+
uint numberOfWaitCycles = 0;
179+
uint blocksProcessed = 0;
179180

180181
var cancellationTokenSource = new CancellationTokenSource();
181182

@@ -189,8 +190,8 @@ public async Task Will_Wait_For_Minimum_Block_Confirmations()
189190
.ReturnsAsync(() => maxBlockNumber);
190191

191192
MockProcessingStrategy
192-
.Setup(s => s.WaitForNextBlock(It.IsAny<int>()))
193-
.Callback<int>((retryNumber) => numberOfWaitCycles++)
193+
.Setup(s => s.WaitForNextBlock(It.IsAny<uint>()))
194+
.Callback<uint>((retryNumber) => numberOfWaitCycles++)
194195
.Returns(Task.CompletedTask);
195196

196197
MockProcessingStrategy
@@ -206,7 +207,7 @@ public async Task Will_Wait_For_Minimum_Block_Confirmations()
206207

207208
Assert.False(result, "Result should be false because execution should have been stopped by cancellation token");
208209

209-
Assert.Equal(1, blocksProcessed);
210+
Assert.Equal((uint)1, blocksProcessed);
210211
Assert.Equal(RequiredBlockConfirmations - 1, numberOfWaitCycles);
211212
Assert.Equal(RequiredBlockConfirmations, maxBlockNumber);
212213
MockProcessingStrategy.VerifyAll();
@@ -247,9 +248,9 @@ public class When_Start_Block_Is_Not_Specified : BlockchainProcessorTests
247248
public async Task
248249
Requests_Last_Block_Processed_From_Strategy_And_Uses_The_Previous_Block_Number()
249250
{
250-
const long LastBlockProcessed = 11;
251-
const long ExpectedStartBlock = LastBlockProcessed - 1;
252-
const long EndingBlock = ExpectedStartBlock;
251+
const ulong LastBlockProcessed = 11;
252+
const ulong ExpectedStartBlock = LastBlockProcessed - 1;
253+
const ulong EndingBlock = ExpectedStartBlock;
253254

254255
MockProcessingStrategy
255256
.Setup(s => s.GetLastBlockProcessedAsync())
@@ -267,9 +268,9 @@ public async Task
267268
[Fact]
268269
public async Task Last_Block_Processsed_Is_Only_Used_If_Greater_Than_Minimum_Block_Number()
269270
{
270-
const long LastBlockProcessed = 11;
271-
const long MinimumStartingBlock = 20;
272-
const long EndingBlock = MinimumStartingBlock;
271+
const ulong LastBlockProcessed = 11;
272+
const ulong MinimumStartingBlock = 20;
273+
const ulong EndingBlock = MinimumStartingBlock;
273274

274275
MockProcessingStrategy
275276
.SetupGet(s => s.MinimumBlockNumber)

0 commit comments

Comments
 (0)