Skip to content

Commit e17c8dc

Browse files
iskydiskyd
committed
refactoring indexer
1 parent 4bcfc98 commit e17c8dc

File tree

6 files changed

+129
-129
lines changed

6 files changed

+129
-129
lines changed

‎flake.nix

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
nativeBuildInputs = [
2525
pkgs.zls
2626
pkgs.zig
27+
pkgs.sqlite
2728
config.pre-commit.settings.enabledPackages
2829
];
2930
shellHook = ''

‎src/db/db.zig

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const sqlite = @import("sqlite");
33
const Output = @import("../tx.zig").Output;
44
const Outpoint = @import("../tx.zig").Outpoint;
55
const Input = @import("../tx.zig").Input;
6+
const Transaction = @import("../tx.zig").Transaction;
67
const KeyPath = @import("../keypath.zig").KeyPath;
78
const Descriptor = @import("../keypath.zig").Descriptor;
89
const Block = @import("../block.zig").Block;
@@ -133,6 +134,7 @@ pub fn getOutput(allocator: std.mem.Allocator, db: *sqlite.Db, txid: [32]u8, vou
133134
if (row != null) {
134135
defer allocator.free(row.?.path);
135136
return Output{
137+
.txid = try utils.hexToBytes(32, &row.?.txid),
136138
.outpoint = Outpoint{ .txid = try utils.hexToBytes(32, &row.?.txid), .vout = row.?.vout },
137139
.amount = row.?.amount,
138140
.unspent = row.?.unspent,
@@ -305,7 +307,7 @@ pub fn getUnspentOutputs(allocator: std.mem.Allocator, db: *sqlite.Db) ![]Output
305307

306308
var outputs = try allocator.alloc(Output, rows.len);
307309
for (rows, 0..) |row, i| {
308-
outputs[i] = Output{ .outpoint = Outpoint{ .txid = try utils.hexToBytes(32, &row.txid), .vout = row.vout }, .amount = row.amount, .unspent = row.unspent, .keypath = try KeyPath(5).fromStr(row.path) };
310+
outputs[i] = Output{ .txid = try utils.hexToBytes(32, &row.txid), .outpoint = Outpoint{ .txid = try utils.hexToBytes(32, &row.txid), .vout = row.vout }, .amount = row.amount, .unspent = row.unspent, .keypath = try KeyPath(5).fromStr(row.path) };
309311
}
310312
return outputs;
311313
}

‎src/indexer.zig

+99-124
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,19 @@ const clap = @import("clap");
1414
const rpc = @import("rpc/rpc.zig");
1515
const db = @import("db/db.zig");
1616
const sqlite = @import("sqlite");
17+
const Thread = std.Thread;
18+
const WaitGroup = Thread.WaitGroup;
19+
20+
const KEYS_GAP = 100;
21+
22+
fn generateKeys(pubkeys: *std.AutoHashMap([20]u8, keypath.KeyPath(5)), kp: keypath.KeyPath(4), extended_pubkey: bip32.ExtendedPublicKey, start: usize, end: usize) !void {
23+
for (start..end) |i| {
24+
const index = @as(u32, @intCast(i));
25+
const pubkey = try bip32.deriveChildFromExtendedPublicKey(extended_pubkey, index);
26+
const pubkey_hash = try pubkey.key.toHash();
27+
try pubkeys.put(pubkey_hash, kp.extendPath(index, false));
28+
}
29+
}
1730

1831
pub fn main() !void {
1932
std.debug.print("WALL-E. Bitcoin Wallet written in Zig.\nIndexer...\n", .{});
@@ -87,7 +100,7 @@ pub fn main() !void {
87100
}
88101
}
89102

90-
const block_height = try rpc.getBlockCount(allocator, &client, rpc_location, auth);
103+
var block_height = try rpc.getBlockCount(allocator, &client, rpc_location, auth);
91104
std.debug.print("Total blocks {d}\n", .{block_height});
92105
std.debug.print("Current blocks {?d}\n", .{current_block_height});
93106
if (current_block_height != null and block_height == current_block_height.?) {
@@ -109,8 +122,17 @@ pub fn main() !void {
109122
// use hashmap to store public key hash for fast check
110123
var pubkeys = std.AutoHashMap([20]u8, keypath.KeyPath(5)).init(allocator);
111124
defer pubkeys.deinit();
125+
126+
// remove, useless
112127
var keypaths = std.AutoHashMap(keypath.KeyPath(5), keypath.Descriptor).init(allocator);
113128
defer keypaths.deinit();
129+
130+
// this is used to store the last used index for every keypath (both internal and external) for every descriptor.
131+
// it is necessary to keep track of the gap between the last generated key and the last used index
132+
// if gap < KEYS_GAP, generate missing keys
133+
var keypath_last_used_index = std.AutoHashMap(keypath.KeyPath(4), u32).init(allocator);
134+
defer keypath_last_used_index.deinit();
135+
114136
var descriptors_map = std.AutoHashMap(keypath.KeyPath(3), [111]u8).init(allocator);
115137
defer descriptors_map.deinit();
116138
for (descriptors) |descriptor| {
@@ -123,47 +145,30 @@ pub fn main() !void {
123145
// For every descriptor we get the latest used index (if one) and generate all the keys to this latest index + 1
124146
// Otherwise we only derive the first keypath (.../0)
125147
for (descriptors) |descriptor| {
126-
const keypath_cap = descriptor.keypath.getStrCap(null) + 2; // + 2 for /0 or /1 (internal / external)
127-
const keypath_internal_str = try allocator.alloc(u8, keypath_cap);
128-
defer allocator.free(keypath_internal_str);
129-
const keypath_external_str = try allocator.alloc(u8, keypath_cap);
130-
defer allocator.free(keypath_external_str);
131-
const partial_keypath_str = try descriptor.keypath.toStr(allocator, null);
132-
defer allocator.free(partial_keypath_str);
133-
_ = try std.fmt.bufPrint(keypath_internal_str, "{s}/{d}", .{ partial_keypath_str, keypath.change_internal_chain });
134-
_ = try std.fmt.bufPrint(keypath_external_str, "{s}/{d}", .{ partial_keypath_str, keypath.change_external_chain });
135-
136-
const last_internal_index = try db.getLastUsedIndexFromOutputs(&database, keypath_internal_str);
137-
const last_external_index = try db.getLastUsedIndexFromOutputs(&database, keypath_internal_str);
138-
var last_indexes: [2]i64 = [2]i64{ -1, -1 };
139-
if (last_internal_index != null) {
140-
last_indexes[0] = last_internal_index.?;
141-
}
142-
if (last_external_index != null) {
143-
last_indexes[1] = last_external_index.?;
144-
}
145-
146-
for (last_indexes, 0..) |last_index, t| {
147-
// This depends on the order specified above. Improve this behaviour pls.
148-
const change_type: u32 = if (t == 0) keypath.change_internal_chain else keypath.change_external_chain;
149-
for (0..@as(usize, @intCast(last_index + 2))) |i| {
150-
const pubkey_addr = descriptors_map.get(descriptor.keypath);
151-
if (pubkey_addr == null) {
152-
std.debug.print("descriptor not found for path {d}'/{d}'/{d}'\n", .{ descriptor.keypath.path[0].value, descriptor.keypath.path[1].value, descriptor.keypath.path[2].value });
153-
continue;
154-
}
155-
156-
const kp = keypath.KeyPath(5){ .path = [5]keypath.KeyPathElement{ descriptor.keypath.path[0], descriptor.keypath.path[1], descriptor.keypath.path[2], keypath.KeyPathElement{ .value = change_type, .is_hardened = false }, keypath.KeyPathElement{ .value = @as(u32, @intCast(i)), .is_hardened = false } } };
157-
158-
const extended_pubkey = try ExtendedPublicKey.fromAddress(pubkey_addr.?);
159-
try keypaths.put(kp, keypath.Descriptor{ .extended_key = pubkey_addr.?, .keypath = descriptor.keypath, .private = false });
160-
161-
const account_kp = keypath.KeyPath(2){ .path = [2]keypath.KeyPathElement{ keypath.KeyPathElement{ .value = kp.path[3].value, .is_hardened = false }, keypath.KeyPathElement{ .value = kp.path[4].value, .is_hardened = false } } };
162-
const pubkey = try bip32.deriveChildFromKeyPath(bip32.ExtendedPublicKey, extended_pubkey, 2, account_kp);
163-
const pubkey_hash = try pubkey.key.toHash();
164-
try pubkeys.put(pubkey_hash, kp);
165-
}
166-
}
148+
const internal_keypath = descriptor.keypath.extendPath(keypath.internal_chain, false);
149+
const external_keypath = descriptor.keypath.extendPath(keypath.external_chain, false);
150+
const internal_keypath_str = try internal_keypath.toStr(allocator, null);
151+
defer allocator.free(internal_keypath_str);
152+
const external_keypath_str = try external_keypath.toStr(allocator, null);
153+
defer allocator.free(external_keypath_str);
154+
155+
const last_internal_index = try db.getLastUsedIndexFromOutputs(&database, internal_keypath_str);
156+
const last_external_index = try db.getLastUsedIndexFromOutputs(&database, external_keypath_str);
157+
158+
const descriptor_pubkey_addr = descriptors_map.get(descriptor.keypath).?;
159+
const descriptor_pubkey = try ExtendedPublicKey.fromAddress(descriptor_pubkey_addr);
160+
const internal_pubkey = try bip32.deriveChildFromExtendedPublicKey(descriptor_pubkey, keypath.internal_chain);
161+
const external_pubkey = try bip32.deriveChildFromExtendedPublicKey(descriptor_pubkey, keypath.external_chain);
162+
163+
const new_internal_last_index = if (last_internal_index == null) KEYS_GAP else last_internal_index.? + KEYS_GAP;
164+
const new_external_last_index = if (last_external_index == null) KEYS_GAP else last_external_index.? + KEYS_GAP;
165+
166+
// 100 gap
167+
try generateKeys(&pubkeys, internal_keypath, internal_pubkey, 0, new_internal_last_index);
168+
try generateKeys(&pubkeys, external_keypath, external_pubkey, 0, new_external_last_index);
169+
170+
try keypath_last_used_index.put(internal_keypath, new_internal_last_index);
171+
try keypath_last_used_index.put(external_keypath, new_external_last_index);
167172
}
168173

169174
std.debug.print("keys generated\n", .{});
@@ -173,94 +178,60 @@ pub fn main() !void {
173178
defer progressbar.end();
174179

175180
const start: usize = if (current_block_height == null) 0 else current_block_height.? + 1;
181+
// align to the current block
182+
var i: usize = start;
183+
while (true) {
184+
if (i >= block_height) {
185+
block_height = try rpc.getBlockCount(allocator, &client, res.args.location.?, auth);
186+
}
176187

177-
for (start..block_height + 1) |i| {
178-
// [72]u8 is for txid + vout in hex format
179-
var outputs = std.ArrayList(Output).init(aa);
180-
// [64]u8 is for txid, bool is for isCoinbase
181-
var relevant_transactions = std.AutoHashMap([32]u8, bool).init(aa);
182188
const blockhash = try rpc.getBlockHash(allocator, &client, res.args.location.?, auth, i);
183-
184189
const raw_transactions = try rpc.getBlockRawTx(aa, &client, res.args.location.?, auth, blockhash);
185190
var raw_transactions_map = std.AutoHashMap([32]u8, []u8).init(aa);
186191

187-
const block_transactions = try aa.alloc(tx.Transaction, raw_transactions.len);
188-
for (raw_transactions, 0..) |tx_raw, j| {
192+
var block_transactions = std.AutoHashMap([32]u8, tx.Transaction).init(aa);
193+
for (raw_transactions) |tx_raw| {
189194
const transaction = try tx.decodeRawTx(aa, tx_raw);
190-
block_transactions[j] = transaction;
195+
try raw_transactions_map.put(try transaction.txid(), tx_raw);
196+
try block_transactions.put(try transaction.txid(), transaction);
191197
}
192198

193-
// Following BIP44 a wallet must not allow the derivation of a new address if the previous one is not used
194-
// So we start by creating 1 new key (both internal and external)
195-
// Everytime we found a new output we need to generate the next key and re-index the same block to be sure all outputs are included
196-
while (true) blk: {
197-
for (block_transactions, 0..) |transaction, k| {
198-
const raw = raw_transactions[k];
199-
const txid = try transaction.txid();
200-
try raw_transactions_map.put(txid, raw);
201-
202-
const txoutputs = try getOutputsFor(aa, transaction, pubkeys);
203-
if (txoutputs.items.len == 0) {
204-
continue;
205-
}
206-
207-
_ = try relevant_transactions.getOrPutValue(txid, transaction.isCoinbase());
208-
209-
for (0..txoutputs.items.len) |j| {
210-
const txoutput = txoutputs.items[j];
211-
// We need to generate the key with idx + n if it doesnt already exists
212-
const kp = txoutput.keypath.?;
213-
const next = kp.getNext(1);
214-
const existing = keypaths.get(next);
215-
216-
// If we are generating a new key and the output is new we start re-indexing the block
217-
// This ensure the fact that we collect all the outputs since the new key could have been used in previous tx in the same block
218-
//var vout_hex: [8]u8 = undefined;
219-
//try utils.intToHexStr(u32, txoutput.outpoint.vout, &vout_hex);
220-
//var key: [72]u8 = undefined;
221-
//_ = try std.fmt.bufPrint(&key, "{s}{s}", .{ try utils.bytesToHex(64, &txoutput.outpoint.txid), vout_hex });
222-
if (existing == null) {
223-
const descriptor = keypaths.get(kp);
224-
// Generate the next key
225-
const account_pubkey = try ExtendedPublicKey.fromAddress(descriptor.?.extended_key);
226-
const account_kp = keypath.KeyPath(2){ .path = [2]keypath.KeyPathElement{ keypath.KeyPathElement{ .value = next.path[3].value, .is_hardened = false }, keypath.KeyPathElement{ .value = next.path[4].value, .is_hardened = false } } };
227-
const next_pubkey = try bip32.deriveChildFromKeyPath(bip32.ExtendedPublicKey, account_pubkey, 2, account_kp);
228-
const next_pubkey_hash = try next_pubkey.key.toHash();
229-
try pubkeys.put(next_pubkey_hash, next);
230-
try keypaths.put(next, descriptor.?);
231-
break :blk;
232-
}
233-
}
234-
235-
for (0..txoutputs.items.len) |j| {
236-
try outputs.append(txoutputs.items[j]);
237-
}
199+
const tx_outputs = try getOutputsFor(aa, block_transactions, pubkeys);
200+
const tx_inputs = try getInputsFor(aa, block_transactions, pubkeys);
201+
202+
for (tx_outputs.items) |tx_output| {
203+
const kp = tx_output.keypath.?.truncPath(4);
204+
const current_last_used = keypath_last_used_index.get(kp).?;
205+
if (tx_output.keypath.?.path[4].value > current_last_used) {
206+
try keypath_last_used_index.put(kp, tx_output.keypath.?.path[4].value);
238207
}
239-
break;
240208
}
241209

242-
const tx_inputs = try getInputsFor(aa, block_transactions, pubkeys);
243-
defer tx_inputs.deinit();
244-
245-
if (outputs.items.len > 0) {
246-
try db.saveOutputs(aa, &database, outputs.items);
210+
if (tx_outputs.items.len > 0) {
211+
try db.saveOutputs(aa, &database, tx_outputs.items);
247212
}
248213
if (tx_inputs.items.len > 0) {
249214
try db.saveInputsAndMarkOutputs(&database, tx_inputs.items);
250215
}
251-
if (relevant_transactions.count() > 0) {
252-
var it = relevant_transactions.keyIterator();
253-
while (it.next()) |txid| {
254-
const raw = raw_transactions_map.get(txid.*).?;
255-
const is_coinbase = relevant_transactions.get(txid.*).?;
256-
try db.saveTransaction(&database, txid.*, raw, is_coinbase, i);
257-
}
216+
for (tx_outputs.items) |tx_output| {
217+
const transaction = block_transactions.get(tx_output.txid).?;
218+
const raw_tx = raw_transactions_map.get(tx_output.txid).?;
219+
try db.saveTransaction(&database, tx_output.txid, raw_tx, transaction.isCoinbase(), i);
258220
}
221+
259222
// Since db writes are not in a single transaction we commit block as latest so that if we restart we dont't risk loosing information, once block is persisted we are sure outputs, inputs and relevant transactions in that block are persisted too. We can recover from partial commit simply reindexing the block.
260223
try db.saveBlock(&database, try utils.hexToBytes(32, &blockhash), i);
261224

262225
progressbar.completeOne();
263226
_ = arena.reset(.free_all);
227+
228+
if (block_height == i) {
229+
break;
230+
}
231+
232+
// Generate new keys if needed
233+
234+
i += 1;
264235
}
265236

266237
std.debug.print("indexing completed\n", .{});
@@ -276,29 +247,33 @@ fn scriptPubkeyToPubkeyHash(allocator: std.mem.Allocator, output: tx.TxOutput) !
276247
return null;
277248
}
278249

279-
fn getOutputsFor(allocator: std.mem.Allocator, transaction: tx.Transaction, pubkeys: std.AutoHashMap([20]u8, keypath.KeyPath(5))) !std.ArrayList(Output) {
250+
fn getOutputsFor(allocator: std.mem.Allocator, transactions: std.AutoHashMap([32]u8, tx.Transaction), pubkeys: std.AutoHashMap([20]u8, keypath.KeyPath(5))) !std.ArrayList(Output) {
280251
var outputs = std.ArrayList(Output).init(allocator);
281-
for (0..transaction.outputs.items.len) |i| {
282-
const tx_output = transaction.outputs.items[i];
283-
const output_pubkey_hash = try scriptPubkeyToPubkeyHash(allocator, tx_output);
284-
if (output_pubkey_hash == null) {
285-
break;
286-
}
252+
var it = transactions.valueIterator();
253+
while (it.next()) |transaction| {
254+
for (0..transaction.outputs.items.len) |i| {
255+
const tx_output = transaction.outputs.items[i];
256+
const output_pubkey_hash = try scriptPubkeyToPubkeyHash(allocator, tx_output);
257+
if (output_pubkey_hash == null) {
258+
continue;
259+
}
287260

288-
const pubkey = pubkeys.get(output_pubkey_hash.?);
289-
if (pubkey != null) {
290-
const txid = try transaction.txid();
291-
const vout = @as(u32, @intCast(i));
292-
const output = Output{ .outpoint = Outpoint{ .txid = txid, .vout = vout }, .keypath = pubkey.?, .amount = tx_output.amount, .unspent = true };
293-
try outputs.append(output);
261+
const pubkey = pubkeys.get(output_pubkey_hash.?);
262+
if (pubkey != null) {
263+
const txid = try transaction.txid();
264+
const vout = @as(u32, @intCast(i));
265+
const output = Output{ .outpoint = Outpoint{ .txid = txid, .vout = vout }, .txid = txid, .keypath = pubkey.?, .amount = tx_output.amount, .unspent = true };
266+
try outputs.append(output);
267+
}
294268
}
295269
}
296270
return outputs;
297271
}
298272

299-
fn getInputsFor(allocator: std.mem.Allocator, transactions: []tx.Transaction, pubkeys: std.AutoHashMap([20]u8, keypath.KeyPath(5))) !std.ArrayList(Input) {
273+
fn getInputsFor(allocator: std.mem.Allocator, transactions: std.AutoHashMap([32]u8, tx.Transaction), pubkeys: std.AutoHashMap([20]u8, keypath.KeyPath(5))) !std.ArrayList(Input) {
300274
var inputs = std.ArrayList(Input).init(allocator);
301-
for (transactions) |transaction| {
275+
var it = transactions.valueIterator();
276+
while (it.next()) |transaction| {
302277
if (transaction.witness.items.len == 0) {
303278
// No segwit, skip
304279
return inputs;

‎src/keypath.zig

+23-2
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ const utils = @import("utils.zig");
55

66
pub const bip_44_purpose = 44;
77
pub const bip_84_purpose = 84; // Segwit
8-
pub const change_external_chain = 0; // Address visible outside the wallet
9-
pub const change_internal_chain = 1; // Not visible outside the wallet, return transaction change
8+
pub const external_chain = 0; // Address visible outside the wallet
9+
pub const internal_chain = 1; // Not visible outside the wallet, return transaction change
1010
pub const bitcoin_coin_type = 0;
1111
pub const bitcoin_testnet_coin_type = 1;
1212

@@ -117,6 +117,27 @@ pub fn KeyPath(comptime depth: u8) type {
117117
path[depth - 1].value = path[depth - 1].value + v;
118118
return Self{ .path = path };
119119
}
120+
121+
pub fn extendPath(self: Self, v: u32, is_hardened: bool) KeyPath(depth + 1) {
122+
var new_path: [depth + 1]KeyPathElement = undefined;
123+
for (self.path, 0..) |p, i| {
124+
new_path[i] = KeyPathElement{ .value = p.value, .is_hardened = p.is_hardened };
125+
}
126+
127+
new_path[depth] = KeyPathElement{ .value = v, .is_hardened = is_hardened };
128+
return KeyPath(depth + 1){ .path = new_path };
129+
}
130+
131+
pub fn truncPath(self: Self, comptime new_depth: u8) KeyPath(new_depth) {
132+
comptime assert(new_depth <= depth);
133+
134+
var new_path: [new_depth]KeyPathElement = undefined;
135+
for (0..new_depth) |i| {
136+
new_path[i] = KeyPathElement{ .value = self.path[i].value, .is_hardened = self.path[i].is_hardened };
137+
}
138+
139+
return KeyPath(new_depth){ .path = new_path };
140+
}
120141
};
121142
}
122143

‎src/tx.zig

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ pub const Input = struct {
3030
pub const Output = struct {
3131
outpoint: Outpoint,
3232
amount: u64,
33+
txid: [32]u8,
3334
unspent: ?bool = null,
3435
keypath: ?KeyPath(5) = null,
3536
};

0 commit comments

Comments
 (0)