-
Notifications
You must be signed in to change notification settings - Fork 4
/
backend-etl.js
148 lines (112 loc) · 4.03 KB
/
backend-etl.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
const fs = require('fs');
const db = require('./db');
const redis = require('redis');
const config = require('./config');
const detectors = require('./detectors');
const { spawn } = require('child_process');
const { chunksToLinesAsync } = require('@rauschma/stringio');
const rc = redis.createClient({
host: config.redis.host,
port: config.redis.port
});
const allowedSupply = (height) => {
let reward = 5000000000n; // 50 BTC
for (let x=1; x<(BigInt(height) / 210000n) + 1n; x++)
reward = reward / 2n;
return reward;
}
const processBlock = async (new_block, current_block) => {
if (current_block) {
current_block.supply_loss = current_block.transactional_loss > 0n || detectors.blockLoss(current_block);
if (current_block.number > 0) {
const previous_block = await db.getBlock(current_block.number - 1);
current_block.current_total_supply = BigInt(previous_block.current_total_supply) + BigInt(current_block.new_supply);
}
else
current_block.current_total_supply = BigInt(current_block.new_supply);
await db.upsertBlock(current_block);
await db.commit();
if (current_block.supply_loss)
rc.publish('block-loss', JSON.stringify(current_block, (k, v) => typeof v === 'bigint' ? v.toString() : v));
}
new_block.input_sum = 0n;
new_block.output_sum = 0n,
new_block.fee_sum = 0n;
new_block.transactional_loss = 0n;
new_block.allowed_supply = allowedSupply(new_block.number);
await db.begin();
await db.upsertBlock(new_block);
return new_block;
}
const processTransaction = async (tx, block) => {
let loss_in_this_transaction = 0n;
for (let t=0; t<tx.inputs.length; t++)
block.input_sum += BigInt(tx.inputs[t].value);
for (let t=0; t<tx.outputs.length; t++) {
if (tx.outputs[t].value > 0) {
if (detectors.outputLoss(block, tx, tx.outputs[t])) {
tx.outputs[t].supply_loss = true;
loss_in_this_transaction += BigInt(tx.outputs[t].value);
}
else
block.output_sum += BigInt(tx.outputs[t].value);
}
}
block.fee_sum += BigInt(tx.fee);
block.new_supply = block.output_sum - block.input_sum;
block.transactional_loss += loss_in_this_transaction;
if (loss_in_this_transaction > 0n) {
await db.upsertTransaction(tx);
await db.upsertInputs(tx.hash, tx.inputs);
await db.upsertOutputs(tx.hash, tx.outputs);
rc.publish('transaction-loss', JSON.stringify(tx, (k, v) => typeof v === 'bigint' ? v.toString() : v));
}
return block;
}
const processReadable = async (readable) => {
let current_block;
await db.begin();
for await (const line of chunksToLinesAsync(readable)) {
readable.pause()
try {
const o = JSON.parse(line);
if (o.type == 'block')
current_block = await processBlock(o, current_block);
else if (o.type == 'transaction')
current_block = await processTransaction(o, current_block);
}
catch (e) {
console.error(e);
}
readable.resume();
}
}
const launchBitcoinETL = async (startblock) => {
const providerURI = `http://${config.bitcoinRPC.username}:${config.bitcoinRPC.password}@${config.bitcoinRPC.host}:${config.bitcoinRPC.port}`;
try {
fs.unlinkSync('last_synced_block.txt');
}
catch (e) {
}
const bitcoinetl = spawn('bitcoinetl', [
'stream', '--chain', 'bitcoin',
'--start-block', startblock,
'--block-batch-size', 1, // we depend on getting one block's transactions at a time in the stream
'--provider-uri', providerURI,
'--lag', 6 // lag by 6 blocks so we don't ingest minor reorgs
], { stdio: ['ignore', 'pipe', process.stderr]} );
await processReadable(bitcoinetl.stdout);
}
const main = async () => {
await db.connect();
let start_block = process.argv[2];
if (!start_block)
start_block = await db.getLatestBlock() + 1;
//start_block = 91722; // duplicate transactionid
//start_block = 124724; // Miner block loss
//start_block = 150951; // MtGox tx loss
//start_block = 640862; // OP_RETURN tx loss
console.log('starting at block number', start_block);
launchBitcoinETL(start_block);
}
main();