Returns KeyPathInformation.Index and reuse queries (#501)

Author: NicolasDorier

NBXplorer.Client/Models/KeyPathInformation.cs

@@ -6,21 +6,6 @@ namespace NBXplorer.Models
 {
 	public class KeyPathInformation
 	{
-		public KeyPathInformation()
-		{
-
-		}
-		
-		public KeyPathInformation(Derivation derivation, DerivationSchemeTrackedSource derivationStrategy, DerivationFeature feature, KeyPath keyPath, NBXplorerNetwork network)
-		{
-			ScriptPubKey = derivation.ScriptPubKey;
-			Redeem = derivation.Redeem;
-			TrackedSource = derivationStrategy;
-			DerivationStrategy = derivationStrategy.DerivationStrategy;
-			Feature = feature;
-			KeyPath = keyPath;
-			Address =  network.CreateAddress(derivationStrategy.DerivationStrategy, keyPath, ScriptPubKey);
-		}
 		public TrackedSource TrackedSource { get; set; }
 		[JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]
 		public DerivationFeature Feature
@@ -48,9 +33,7 @@ public Script Redeem
 		{
 			get; set;
 		}
-		public int GetIndex(KeyPathTemplates keyPathTemplates)
-		{
-			return (int)keyPathTemplates.GetKeyPathTemplate(Feature).GetIndex(KeyPath);
-		}
+		[JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)]
+		public int? Index { get; set; }
 	}
 }

NBXplorer.Tests/UnitTest1.cs

@@ -2643,7 +2643,7 @@ public async Task CanReserveAddress()
 				await Task.WhenAll(tasks.ToArray());
 
 				var paths = tasks.Select(t => t.Result).ToDictionary(c => c.KeyPath);
-				Assert.Equal(99, paths.Select(p => p.Value.GetIndex(KeyPathTemplates.Default)).Max());
+				Assert.Equal(99, paths.Select(p => p.Value.Index).Max());
 
 				tester.Client.CancelReservation(bob, new[] { new KeyPath("0/0") });
 				var addr = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit);
@@ -2658,6 +2658,7 @@ public async Task CanReserveAddress()
 				tester.Client.CancelReservation(bob, new[] { new KeyPath("0/0") });
 				addr2 = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit);
 				Assert.Equal(new KeyPath("0/100"), addr2.KeyPath);
+				Assert.Equal(100, addr2.Index);
 			}
 		}
 
@@ -2854,12 +2855,15 @@ public async Task CanTrack5()
 
 				Logs.Tester.LogInformation("Let's check if direct addresses can be tracked by sending to 0");
 				var address = await tester.Client.GetUnusedAsync(pubkey, DerivationFeature.Direct);
+				Assert.Equal(0, address.Index);
 				Assert.Equal(DerivationFeature.Direct, address.Feature);
 				fundingTx = tester.SendToAddress(tester.AddressOf(key, "0"), Money.Coins(1.0m));
 				tester.Notifications.WaitForTransaction(pubkey, fundingTx);
 				utxo = tester.Client.GetUTXOs(pubkey);
 				Assert.Equal(address.ScriptPubKey, utxo.Unconfirmed.UTXOs[0].ScriptPubKey);
 				var address2 = await tester.Client.GetUnusedAsync(pubkey, DerivationFeature.Direct);
+				Logs.Tester.LogInformation("ADDRESS2: " + address2.Address);
+				Assert.Equal(1, address2.Index);
 				Assert.Equal(new KeyPath(1), address2.KeyPath);
 
 				Logs.Tester.LogInformation("Let's check see if an unconf tx can be conf then unconf again");
@@ -3680,7 +3684,7 @@ public async Task CanScanUTXOSet()
 				// Nothing has been tracked because it is way out of bound and the first address is always unused
 				var transactions = tester.Client.GetTransactions(pubkey);
 				Assert.Empty(transactions.ConfirmedTransactions.Transactions);
-				Assert.Equal(0, tester.Client.GetUnused(pubkey, DerivationFeature.Deposit).GetIndex(KeyPathTemplates.Default));
+				Assert.Equal(0, tester.Client.GetUnused(pubkey, DerivationFeature.Deposit).Index);
 
 				// W00t! let's scan and see if it now appear in the UTXO
 				tester.Client.ScanUTXOSet(pubkey, batchsize, gaplimit);
@@ -3703,7 +3707,7 @@ public async Task CanScanUTXOSet()
 #pragma warning restore CS0618 // Type or member is obsolete
 
 				Logs.Tester.LogInformation($"Check that the address pool has been emptied: 0/51 should be the next unused address");
-				Assert.Equal(51, tester.Client.GetUnused(pubkey, DerivationFeature.Deposit).GetIndex(KeyPathTemplates.Default));
+				Assert.Equal(51, tester.Client.GetUnused(pubkey, DerivationFeature.Deposit).Index);
 				utxo = tester.Client.GetUTXOs(pubkey);
 				Assert.Equal(txId, utxo.Confirmed.UTXOs[0].TransactionHash);
 

NBXplorer/Backend/Repository.cs

@@ -18,6 +18,7 @@
 using System.Text.RegularExpressions;
 using Npgsql;
 using static NBXplorer.Backend.DbConnectionHelper;
+using Microsoft.AspNetCore.DataProtection.KeyManagement;
 
 
 namespace NBXplorer.Backend
@@ -368,32 +369,94 @@ public async Task<MultiValueDictionary<Script, KeyPathInformation>> GetKeyInform
 			await using var connection = await connectionFactory.CreateConnection();
 			return await GetKeyInformations(connection, scripts);
 		}
-		async Task<MultiValueDictionary<Script, KeyPathInformation>> GetKeyInformations(DbConnection connection, IList<Script> scripts)
+		abstract class GetKeyInformationsQuery
 		{
-			scripts = scripts.Distinct().ToArray();
-			MultiValueDictionary<Script, KeyPathInformation> result = new MultiValueDictionary<Script, KeyPathInformation>();
-			foreach (var s in scripts)
-				result.AddRange(s, Array.Empty<KeyPathInformation>());
-			var command = connection.CreateCommand();
-			if (scripts.Count == 0)
+			public static GetKeyInformationsQuery ByScripts(IList<Script> scripts) => new ByScriptsQuery(scripts);
+			public static GetKeyInformationsQuery ByUnused(DerivationStrategyBase strategy, DerivationFeature derivationFeature, int skip, NBXplorerNetwork network) => new ByUnusedQuery(strategy, derivationFeature, skip, network);
+			public abstract string GetScriptsQuery();
+			public virtual string GetKeyPathInfoPredicate() => string.Empty;
+			public abstract void AddParameters(DynamicParameters parameters);
+			public virtual bool IsEmpty => false;
+			public virtual MultiValueDictionary<Script, KeyPathInformation> CreateResult() => new();
+			class ByUnusedQuery : GetKeyInformationsQuery
+			{
+				private int n;
+				private readonly string descriptorId;
+
+				public ByUnusedQuery(DerivationStrategyBase strategy, DerivationFeature derivationFeature, int n, NBXplorerNetwork network)
+				{
+					this.n = n;
+					descriptorId = DBUtils.nbxv1_get_descriptor_id(network.CryptoCode, strategy.ToString(), derivationFeature.ToString());
+				}
+
+				public override void AddParameters(DynamicParameters parameters)
+				{
+					parameters.Add("skip", n);
+					parameters.Add("descriptor", descriptorId);
+				}
+				public override string GetScriptsQuery()
+				{
+					return $@"
+						(SELECT script FROM descriptors_scripts_unused
+						WHERE code=@code AND descriptor=@descriptor
+						ORDER BY idx
+						LIMIT 1 OFFSET @skip) AS r (script)
+					";
+				}
+				public override string GetKeyPathInfoPredicate() => "AND ki.descriptor=@descriptor";
+			}
+			class ByScriptsQuery : GetKeyInformationsQuery
+			{
+				private readonly Script[] scripts;
+
+				public override bool IsEmpty => scripts.Length is 0;
+
+				public ByScriptsQuery(IList<Script> scripts)
+				{
+					this.scripts = scripts.Distinct().ToArray();
+				}
+				public override string GetScriptsQuery() => "unnest(@records) AS r (script)";
+				public override void AddParameters(DynamicParameters parameters)
+				{
+					parameters.Add("records", scripts.Select(s => s.ToHex()).ToArray());
+				}
+				public override MultiValueDictionary<Script, KeyPathInformation> CreateResult()
+				{
+					MultiValueDictionary<Script, KeyPathInformation> result = new MultiValueDictionary<Script, KeyPathInformation>();
+					foreach (var s in scripts)
+						result.AddRange(s, Array.Empty<KeyPathInformation>());
+					return result;
+				}
+			}
+		}
+		Task<MultiValueDictionary<Script, KeyPathInformation>> GetKeyInformations(DbConnection connection, IList<Script> scripts)
+			=> GetKeyInformations(connection, GetKeyInformationsQuery.ByScripts(scripts));
+		async Task<MultiValueDictionary<Script, KeyPathInformation>> GetKeyInformations(DbConnection connection, GetKeyInformationsQuery query)
+		{
+			var result = query.CreateResult();
+			if (query.IsEmpty)
 				return result;
+
+			DynamicParameters parameters = new DynamicParameters();
+			parameters.Add("code", Network.CryptoCode);
+			query.AddParameters(parameters);
 			string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : "";
 			var rows = await connection.QueryAsync($@"
-			    SELECT ts.code, ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn},
+			    SELECT ts.code, ts.script, ts.addr, ts.derivation, ts.keypath, ts.idx, ts.feature, ts.redeem{additionalColumn},
 				       ts.wallet_id,
 				       w.metadata AS wallet_metadata
-				FROM unnest(@records) AS r (script),
+				FROM {query.GetScriptsQuery()},
 				LATERAL (
 				    SELECT code, script, wallet_id, addr, descriptor_metadata->>'derivation' derivation, 
-				           keypath, descriptors_scripts_metadata->>'redeem' redeem, 
+				           keypath, ki.idx, descriptors_scripts_metadata->>'redeem' redeem, 
 				           descriptors_scripts_metadata->>'blindedAddress' blinded_addr, 
 				           descriptors_scripts_metadata->>'blindingKey' blindingKey, 
-				           descriptor_metadata->>'descriptor' descriptor
-				    FROM nbxv1_keypath_info ki 
-				    WHERE ki.code=@code AND ki.script=r.script
+				           descriptor_metadata->>'descriptor' descriptor,
+				           descriptor_metadata->>'feature' feature
+				    FROM nbxv1_keypath_info ki
+				    WHERE ki.code=@code AND ki.script=r.script {query.GetKeyPathInfoPredicate()}
 				) ts
-				JOIN wallets w USING(wallet_id)",
-	new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() });
+				JOIN wallets w USING(wallet_id)", parameters);
 			foreach (var r in rows)
 			{
 				// This might be the case for a derivation added by a different indexer
@@ -425,7 +488,12 @@ JOIN wallets w USING(wallet_id)",
 				ki.KeyPath = keypath;
 				ki.ScriptPubKey = script;
 				ki.TrackedSource = trackedSource;
-				ki.Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath);
+				ki.Feature = DerivationFeature.Deposit;
+				if (keypath is not null)
+				{
+					ki.Feature = Enum.Parse<DerivationFeature>(r.feature, true);
+					ki.Index = (int)r.idx;
+				}
 				ki.Redeem = redeem is null ? null : Script.FromHex(redeem);
 				result.Add(script, ki);
 			}
@@ -783,14 +851,8 @@ public async Task<KeyPathInformation> GetUnused(DerivationStrategyBase strategy,
 		{
 			await using var helper = await connectionFactory.CreateConnectionHelper(Network);
 			var connection = helper.Connection;
-			var key = GetDescriptorKey(strategy, derivationFeature);
-			string additionalColumn = Network.IsElement ? ", ds_metadata->>'blindedAddress' blinded_addr" : string.Empty;
 			retry:
-			var unused = await connection.QueryFirstOrDefaultAsync(
-				$"SELECT script, addr, nbxv1_get_keypath(d_metadata, idx) keypath, ds_metadata->>'redeem' redeem {additionalColumn} FROM descriptors_scripts_unused " +
-				"WHERE code=@code AND descriptor=@descriptor " +
-				"ORDER BY idx " +
-				"LIMIT 1 OFFSET @skip", new { key.code, key.descriptor, skip = n });
+			var unused = (await GetKeyInformations(connection, GetKeyInformationsQuery.ByUnused(strategy, derivationFeature, n, Network))).FirstOrDefault().Value?.FirstOrDefault();
 			if (unused is null)
 			{
 				// If we don't find unused address, then either:
@@ -809,23 +871,12 @@ public async Task<KeyPathInformation> GetUnused(DerivationStrategyBase strategy,
 			}
 			if (reserve)
 			{
-				var updated = await connection.ExecuteAsync("UPDATE descriptors_scripts SET used='t' WHERE code=@code AND script=@script AND descriptor=@descriptor AND used='f'", new { key.code, unused.script, key.descriptor });
+				var updated = await connection.ExecuteAsync("UPDATE descriptors_scripts SET used='t' WHERE code=@code AND script=@script AND descriptor=@descriptor AND used='f'", new { code = Network.CryptoCode, script = unused.ScriptPubKey.ToHex(), descriptor = GetDescriptorKey(strategy, derivationFeature).descriptor });
 				if (updated == 0)
 					goto retry;
 			}
-			var keypath = KeyPath.Parse(unused.keypath);
-			var keyInfo = new KeyPathInformation()
-			{
-				Address = GetAddress(unused),
-				DerivationStrategy = strategy,
-				KeyPath = keypath,
-				ScriptPubKey = Script.FromHex(unused.script),
-				TrackedSource = new DerivationSchemeTrackedSource(strategy),
-				Feature = KeyPathTemplates.GetDerivationFeature(keypath),
-				Redeem = unused.redeem is string s ? Script.FromHex(s) : null
-			};
-			await ImportAddressToRPC(helper, keyInfo.TrackedSource, keyInfo.Address, keyInfo.KeyPath);
-			return keyInfo;
+			await ImportAddressToRPC(helper, unused.TrackedSource, unused.Address, unused.KeyPath);
+			return unused;
 		}
 
 		record SingleAddressInsert(string code, string script, string address, string walletid);
@@ -863,7 +914,7 @@ internal async Task SaveKeyInformations(DbConnection connection, KeyPathInformat
 				{
 					descriptorInsert.Add(new DescriptorScriptInsert(
 						descriptorKey.descriptor,
-						ki.GetIndex(KeyPathTemplates),
+						ki.Index.Value,
 						ki.ScriptPubKey.ToHex(),
 						metadata?.ToString(Formatting.None),
 						addr.ToString(),

NBXplorer/DBScripts/023.KeyPathInfoReturnsIndex.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE VIEW nbxv1_keypath_info AS
+ SELECT ws.code,
+    ws.script,
+    s.addr,
+    d.metadata AS descriptor_metadata,
+    nbxv1_get_keypath(d.metadata, ds.idx) AS keypath,
+    ds.metadata AS descriptors_scripts_metadata,
+    ws.wallet_id,
+    ds.idx,
+    ds.used,
+    d.descriptor
+   FROM ((wallets_scripts ws
+     JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script))))
+     LEFT JOIN ((wallets_descriptors wd
+     JOIN descriptors_scripts ds ON (((ds.code = wd.code) AND (ds.descriptor = wd.descriptor))))
+     JOIN descriptors d ON (((d.code = ds.code) AND (d.descriptor = ds.descriptor)))) ON (((wd.wallet_id = ws.wallet_id) AND (wd.code = ws.code) AND (ds.script = ws.script))));

NBXplorer/DBScripts/FullSchema.sql

@@ -959,7 +959,10 @@ CREATE VIEW nbxv1_keypath_info AS
     d.metadata AS descriptor_metadata,
     nbxv1_get_keypath(d.metadata, ds.idx) AS keypath,
     ds.metadata AS descriptors_scripts_metadata,
-    ws.wallet_id
+    ws.wallet_id,
+    ds.idx,
+    ds.used,
+    d.descriptor
    FROM ((wallets_scripts ws
      JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script))))
      LEFT JOIN ((wallets_descriptors wd
@@ -1367,6 +1370,7 @@ INSERT INTO nbxv1_migrations VALUES ('019.FixDoubleSpendDetection2');
 INSERT INTO nbxv1_migrations VALUES ('020.ReplacingShouldBeIdempotent');
 INSERT INTO nbxv1_migrations VALUES ('021.KeyPathInfoReturnsWalletId');
 INSERT INTO nbxv1_migrations VALUES ('022.WalletsWalletsParentIdIndex');
+INSERT INTO nbxv1_migrations VALUES ('023.KeyPathInfoReturnsIndex');
 
 ALTER TABLE ONLY nbxv1_migrations
     ADD CONSTRAINT nbxv1_migrations_pkey PRIMARY KEY (script_name);

NBXplorer/NBXplorer.csproj

@@ -27,6 +27,7 @@
     <None Remove="DBScripts\018.FastWalletRecent.sql" />
     <None Remove="DBScripts\020.ReplacingShouldBeIdempotent.sql" />
     <None Remove="DBScripts\022.WalletsWalletsParentIdIndex.sql" />
+    <None Remove="DBScripts\023.KeyPathInfoReturnsIndex.sql" />
   </ItemGroup>
   <ItemGroup>
 	<EmbeddedResource Include="DBScripts\*.sql" />
@@ -60,4 +61,5 @@
       <LastGenOutput>Resources.Designer.cs</LastGenOutput>
     </EmbeddedResource>
   </ItemGroup>
+  <ProjectExtensions><VisualStudio><UserProperties wwwroot_4api_1json__JsonSchema="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/refs/tags/3.1.0/schemas/v3.0/schema.json" /></VisualStudio></ProjectExtensions>
 </Project>

NBXplorer/ScanUTXOSetService.cs

@@ -308,9 +308,19 @@ private ScannedItems GetScannedItems(ScanUTXOWorkItem workItem, ScanUTXOProgress
 				Enumerable.Range(progress.From, progress.Count)
 						  .Select(index =>
 						  {
+							  var keyPath = keyPathTemplate.GetKeyPath(index, false);
 							  var derivation = lineDerivation.Derive((uint)index);
-							  var info = new KeyPathInformation(derivation, derivationStrategy, feature,
-								  keyPathTemplate.GetKeyPath(index, false), network);
+							  var info = new KeyPathInformation()
+							  {
+								ScriptPubKey = derivation.ScriptPubKey,
+								DerivationStrategy = derivationStrategy.DerivationStrategy,
+								Feature = feature,
+								KeyPath = keyPath,
+								Redeem = derivation.Redeem,
+								TrackedSource = derivationStrategy,
+								Address = network.CreateAddress(derivationStrategy.DerivationStrategy, keyPath, derivation.ScriptPubKey),
+								Index = index
+							  };
 							  items.Descriptors.Add(OutputDescriptor.NewRaw(info.ScriptPubKey, network.NBitcoinNetwork));
 							  items.KeyPathInformations.TryAdd(info.ScriptPubKey, info);
 							  return info;