diff --git a/CHANGELOG.md b/CHANGELOG.md index af462accf..ff8e1d8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 0.7.0-rc.55 (Synonym Fork) +# 0.7.0-rc.56 (Synonym Fork) ## Bug Fixes @@ -65,6 +65,7 @@ ## Synonym Fork Additions +- Added rolling lookahead for derived on-chain accounts after their initial full scan. - Added configurable Electrum batch size and stop gap for full scans of non-primary on-chain wallets, while preserving the existing primary-wallet behavior and defaults. - Added derived-account lifecycle parity with account-`0` wallets: configured accounts load on @@ -83,9 +84,9 @@ `Config::onchain_wallet_accounts` to load them on each build. BDK data remains persisted per account after unload - Derived accounts full-scan after registration, then use incremental sync. Apps issuing addresses - from an exported xpub reveal their highest issued index before syncing; descriptor origins use - the real account path; channel preflight requires an account-`0` SegWit builder before counting - non-Legacy (including derived) funds + from an exported xpub can reveal their highest issued index to cover gaps beyond the rolling + lookahead; descriptor origins use the real account path; channel preflight requires an account-`0` + SegWit builder before counting non-Legacy (including derived) funds - Bitcoind Listen synchronizes every wallet from its own checkpoint alongside the Lightning listeners, handles shorter and equal-height forked tips, and replays the mempool when the loaded account set changes diff --git a/Cargo.toml b/Cargo.toml index 6cc499858..f60816b70 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["bindings/uniffi-bindgen"] [package] name = "ldk-node" -version = "0.7.0-rc.55" +version = "0.7.0-rc.56" authors = ["Elias Rohrer "] homepage = "https://lightningdevkit.org/" license = "MIT OR Apache-2.0" diff --git a/Package.swift b/Package.swift index a9037e0e4..911b8c1e9 100644 --- a/Package.swift +++ b/Package.swift @@ -3,8 +3,8 @@ import PackageDescription -let tag = "v0.7.0-rc.55" -let checksum = "098d48d602fe000f1a9dbc7e758a0fe7bd8720c12b0036c38edf38283edd55f3" +let tag = "v0.7.0-rc.56" +let checksum = "91a31d12acc142ffaa405f4cae8eda0a414f71cf3bd0c4ef26b420a741483617" let url = "https://github.com/synonymdev/ldk-node/releases/download/\(tag)/LDKNodeFFI.xcframework.zip" let package = Package( diff --git a/bindings/kotlin/ldk-node-android/gradle.properties b/bindings/kotlin/ldk-node-android/gradle.properties index c6f65042c..13ec6e28e 100644 --- a/bindings/kotlin/ldk-node-android/gradle.properties +++ b/bindings/kotlin/ldk-node-android/gradle.properties @@ -3,4 +3,4 @@ android.useAndroidX=true android.enableJetifier=true kotlin.code.style=official group=com.synonym -version=0.7.0-rc.55 +version=0.7.0-rc.56 diff --git a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/arm64-v8a/libldk_node.so b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/arm64-v8a/libldk_node.so index 54a09051e..fb6c4a16f 100755 Binary files a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/arm64-v8a/libldk_node.so and b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/arm64-v8a/libldk_node.so differ diff --git a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/armeabi-v7a/libldk_node.so b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/armeabi-v7a/libldk_node.so index 93b8cbbf7..d986b10b9 100755 Binary files a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/armeabi-v7a/libldk_node.so and b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/armeabi-v7a/libldk_node.so differ diff --git a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/x86_64/libldk_node.so b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/x86_64/libldk_node.so index f9db6b1bb..703e895b9 100755 Binary files a/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/x86_64/libldk_node.so and b/bindings/kotlin/ldk-node-android/lib/src/main/jniLibs/x86_64/libldk_node.so differ diff --git a/bindings/kotlin/ldk-node-jvm/gradle.properties b/bindings/kotlin/ldk-node-jvm/gradle.properties index 70ca71025..72fbd0b07 100644 --- a/bindings/kotlin/ldk-node-jvm/gradle.properties +++ b/bindings/kotlin/ldk-node-jvm/gradle.properties @@ -1,4 +1,4 @@ org.gradle.jvmargs=-Xmx1536m kotlin.code.style=official group=com.synonym -version=0.7.0-rc.55 +version=0.7.0-rc.56 diff --git a/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-aarch64/libldk_node.dylib b/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-aarch64/libldk_node.dylib index 711d07683..13a761ac7 100644 Binary files a/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-aarch64/libldk_node.dylib and b/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-aarch64/libldk_node.dylib differ diff --git a/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-x86-64/libldk_node.dylib b/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-x86-64/libldk_node.dylib index 48414c63b..f9cb199ed 100644 Binary files a/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-x86-64/libldk_node.dylib and b/bindings/kotlin/ldk-node-jvm/lib/src/main/resources/darwin-x86-64/libldk_node.dylib differ diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 57bf0929a..dc9f9f1a4 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ldk_node" -version = "0.7.0-rc.55" +version = "0.7.0-rc.56" authors = [ { name="Elias Rohrer", email="dev@tnull.de" }, ] diff --git a/crates/bdk-wallet-aggregate/src/lib.rs b/crates/bdk-wallet-aggregate/src/lib.rs index 9b84dabdb..6215359f1 100644 --- a/crates/bdk-wallet-aggregate/src/lib.rs +++ b/crates/bdk-wallet-aggregate/src/lib.rs @@ -746,6 +746,39 @@ where Ok(wallet.start_sync_with_revealed_spks().build()) } + /// Build an incremental sync request including external lookahead scripts. + /// + /// The wallet's configured lookahead determines how many unrevealed scripts are included. + pub fn wallet_incremental_sync_request_with_lookahead( + &self, key: &K, + ) -> Result, Error> { + let wallet = self.wallets.get(key).ok_or(Error::WalletNotFound)?; + let mut request = wallet.start_sync_with_revealed_spks(); + let external_lookahead = wallet.spk_index().lookahead(); + if external_lookahead == 0 { + return Ok(request.build()); + } + + let start_index = match wallet.derivation_index(KeychainKind::External) { + Some(BIP32_MAX_NORMAL_INDEX) => return Ok(request.build()), + Some(index) => index + 1, + None => 0, + }; + let end_index = + start_index.saturating_add(external_lookahead - 1).min(BIP32_MAX_NORMAL_INDEX); + let lookahead_spks = (start_index..=end_index) + .map(|index| { + wallet + .spk_index() + .spk_at_index(KeychainKind::External, index) + .map(|spk| ((KeychainKind::External, index), spk)) + .ok_or(Error::WalletOperationFailed) + }) + .collect::, _>>()?; + request = request.spks_with_indexes(lookahead_spks); + Ok(request.build()) + } + /// Apply a chain update to the primary wallet. /// /// Returns the wallet events and a list of all transaction IDs in the @@ -1698,31 +1731,52 @@ mod tests { P: WalletPersister, P::Error: std::fmt::Debug, { - PersistedWallet::create( - persister, - Wallet::create( - Bip84(xprv, KeychainKind::External), - Bip84(xprv, KeychainKind::Internal), - ) - .network(Network::Regtest), + create_empty_wallet_from_xprv_with_lookahead(persister, xprv, None) + } + + fn create_empty_wallet_from_xprv_with_lookahead

( + persister: &mut P, xprv: Xpriv, lookahead: Option, + ) -> PersistedWallet

+ where + P: WalletPersister, + P::Error: std::fmt::Debug, + { + let mut params = Wallet::create( + Bip84(xprv, KeychainKind::External), + Bip84(xprv, KeychainKind::Internal), ) - .unwrap() + .network(Network::Regtest); + if let Some(lookahead) = lookahead { + params = params.lookahead(lookahead); + } + PersistedWallet::create(persister, params).unwrap() } fn load_empty_wallet

(persister: &mut P) -> PersistedWallet

+ where + P: WalletPersister, + P::Error: std::fmt::Debug, + { + load_empty_wallet_with_lookahead(persister, None) + } + + fn load_empty_wallet_with_lookahead

( + persister: &mut P, lookahead: Option, + ) -> PersistedWallet

where P: WalletPersister, P::Error: std::fmt::Debug, { let xprv = test_xprv(); - Wallet::load() + let mut params = Wallet::load() .descriptor(KeychainKind::External, Some(Bip84(xprv, KeychainKind::External))) .descriptor(KeychainKind::Internal, Some(Bip84(xprv, KeychainKind::Internal))) .extract_keys() - .check_network(Network::Regtest) - .load_wallet(persister) - .unwrap() - .expect("wallet should have been persisted") + .check_network(Network::Regtest); + if let Some(lookahead) = lookahead { + params = params.lookahead(lookahead); + } + params.load_wallet(persister).unwrap().expect("wallet should have been persisted") } #[test] @@ -2283,6 +2337,80 @@ mod tests { assert_ne!(next.address, peeked.address); } + #[test] + fn incremental_lookahead_does_not_advance_receive_index() { + let mut persister = NoopPersister; + let wallet = + create_empty_wallet_from_xprv_with_lookahead(&mut persister, test_xprv(), Some(3)); + let mut aggregate = AggregateWallet::::new(wallet, persister, 0, vec![]); + + let request = aggregate.wallet_incremental_sync_request_with_lookahead(&0).unwrap(); + assert_eq!(request.progress().spks_remaining, 3); + assert_eq!(aggregate.new_address_info_for(&0).unwrap().index, 0); + } + + #[test] + fn incremental_lookahead_rolls_forward_after_activity() { + let mut persister = NoopPersister; + let wallet = + create_empty_wallet_from_xprv_with_lookahead(&mut persister, test_xprv(), Some(3)); + let mut aggregate = AggregateWallet::::new(wallet, persister, 0, vec![]); + let address = aggregate.address_info_for(&0, KeychainKind::External, 2).unwrap().address; + let transaction = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { txid: Txid::from_byte_array([0x18; 32]), vout: 0 }, + ..Default::default() + }], + output: vec![TxOut { + value: Amount::from_sat(50_000), + script_pubkey: address.script_pubkey(), + }], + }; + + aggregate.apply_mempool_txs(vec![(transaction, 0)], vec![]).unwrap(); + + let request = aggregate.wallet_incremental_sync_request_with_lookahead(&0).unwrap(); + assert_eq!(request.progress().spks_remaining, 6); + assert_eq!(aggregate.new_address_info_for(&0).unwrap().index, 3); + } + + #[test] + fn incremental_lookahead_activity_persists_the_active_index() { + let mut create_persister = MemoryPersister::default(); + let wallet = create_empty_wallet_from_xprv_with_lookahead( + &mut create_persister, + test_xprv(), + Some(3), + ); + let aggregate_persister = create_persister.clone(); + { + let mut aggregate = + AggregateWallet::::new(wallet, aggregate_persister, 0, vec![]); + let address = + aggregate.address_info_for(&0, KeychainKind::External, 2).unwrap().address; + let transaction = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { txid: Txid::from_byte_array([0x19; 32]), vout: 0 }, + ..Default::default() + }], + output: vec![TxOut { + value: Amount::from_sat(50_000), + script_pubkey: address.script_pubkey(), + }], + }; + aggregate.apply_mempool_txs(vec![(transaction, 0)], vec![]).unwrap(); + } + + let mut reload_persister = create_persister.clone(); + let wallet = load_empty_wallet_with_lookahead(&mut reload_persister, Some(3)); + let mut aggregate = AggregateWallet::::new(wallet, reload_persister, 0, vec![]); + assert_eq!(aggregate.new_address_info_for(&0).unwrap().index, 3); + } + #[test] fn address_info_for_index_supports_internal_keychain_without_advancing_receive_index() { let mut persister = NoopPersister; diff --git a/src/builder.rs b/src/builder.rs index eed4b72cd..1810c9978 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -52,8 +52,8 @@ use crate::config::{ default_user_config, may_announce_channel, AddressType, AddressTypeRuntimeConfig, AnnounceError, AsyncPaymentsRole, BitcoindRestClientConfig, Config, ElectrumSyncConfig, EsploraSyncConfig, OnchainWalletAccount, RuntimeSyncIntervals, ScoringDecayParameters, - ScoringFeeParameters, DEFAULT_ESPLORA_SERVER_URL, DEFAULT_LOG_FILENAME, DEFAULT_LOG_LEVEL, - MAX_ONCHAIN_WALLET_ACCOUNT_INDEX, WALLET_KEYS_SEED_LEN, + ScoringFeeParameters, BDK_CLIENT_STOP_GAP, DEFAULT_ESPLORA_SERVER_URL, DEFAULT_LOG_FILENAME, + DEFAULT_LOG_LEVEL, MAX_ONCHAIN_WALLET_ACCOUNT_INDEX, WALLET_KEYS_SEED_LEN, }; use crate::connection::ConnectionManager; use crate::event::EventQueue; @@ -115,6 +115,18 @@ enum ChainDataSourceConfig { }, } +fn derived_account_lookahead(chain_data_source_config: Option<&ChainDataSourceConfig>) -> u32 { + match chain_data_source_config { + Some(ChainDataSourceConfig::Electrum { sync_config, .. }) => { + sync_config.unwrap_or_default().additional_wallet_full_scan_stop_gap.max(1) + }, + Some(ChainDataSourceConfig::Bitcoind { .. }) => { + bdk_chain::keychain_txout::DEFAULT_LOOKAHEAD + }, + Some(ChainDataSourceConfig::Esplora { .. }) | None => BDK_CLIENT_STOP_GAP as u32, + } +} + #[derive(Debug, Clone)] enum EntropySourceConfig { SeedFile(String), @@ -1350,42 +1362,49 @@ fn account_keychain_derivation_path( /// persisted wallet was restored. pub(crate) fn get_or_create_wallet_for_account( wallet_account: OnchainWalletAccount, master_xprv: Xpriv, network: Network, - persister: &mut KVStoreWalletPersister, + persister: &mut KVStoreWalletPersister, lookahead: Option, ) -> Result<(PersistedWallet, bool), BuildError> { macro_rules! load_or_create { ($ext:expr, $int:expr) => {{ - let wallet_opt = BdkWallet::load() + let mut load_params = BdkWallet::load() .descriptor(KeychainKind::External, Some($ext)) .descriptor(KeychainKind::Internal, Some($int)) .extract_keys() - .check_network(network) - .load_wallet(persister) - .map_err(|e| match e { - bdk_wallet::LoadWithPersistError::InvalidChangeSet( - bdk_wallet::LoadError::Mismatch(bdk_wallet::LoadMismatch::Network { - loaded, - expected, - }), - ) => { - log::error!( - "Failed to setup wallet: Networks do not match. Expected {} but got {}", - expected, - loaded - ); - BuildError::NetworkMismatch - }, - _ => { - log::error!("Failed to set up wallet: {}", e); - BuildError::WalletSetupFailed - }, - })?; + .check_network(network); + if let Some(lookahead) = lookahead { + load_params = load_params.lookahead(lookahead); + } + let wallet_opt = load_params.load_wallet(persister).map_err(|e| match e { + bdk_wallet::LoadWithPersistError::InvalidChangeSet( + bdk_wallet::LoadError::Mismatch(bdk_wallet::LoadMismatch::Network { + loaded, + expected, + }), + ) => { + log::error!( + "Failed to setup wallet: Networks do not match. Expected {} but got {}", + expected, + loaded + ); + BuildError::NetworkMismatch + }, + _ => { + log::error!("Failed to set up wallet: {}", e); + BuildError::WalletSetupFailed + }, + })?; match wallet_opt { Some(w) => Ok((w, true)), - None => BdkWallet::create($ext, $int) - .network(network) - .create_wallet(persister) - .map(|w| (w, false)) - .map_err(|_| BuildError::WalletSetupFailed), + None => { + let mut create_params = BdkWallet::create($ext, $int).network(network); + if let Some(lookahead) = lookahead { + create_params = create_params.lookahead(lookahead); + } + create_params + .create_wallet(persister) + .map(|w| (w, false)) + .map_err(|_| BuildError::WalletSetupFailed) + }, } }}; } @@ -1484,7 +1503,7 @@ fn validate_configured_onchain_wallet_accounts( fn build_wallet_for_account( config: &Config, wallet_account: OnchainWalletAccount, xprv: Xpriv, kv_store: Arc, - logger: Arc, chain_tip_opt: Option<&BestBlock>, + logger: Arc, chain_tip_opt: Option<&BestBlock>, lookahead: Option, ) -> Result< (OnchainWalletAccount, PersistedWallet, KVStoreWalletPersister), BuildError, @@ -1492,12 +1511,17 @@ fn build_wallet_for_account( let mut persister = KVStoreWalletPersister::new(Arc::clone(&kv_store), Arc::clone(&logger), wallet_account); - let (mut wallet, loaded_from_store) = - get_or_create_wallet_for_account(wallet_account, xprv, config.network, &mut persister) - .map_err(|e| { - log_error!(logger, "Failed to setup wallet for {:?}: {}", wallet_account, e); - e - })?; + let (mut wallet, loaded_from_store) = get_or_create_wallet_for_account( + wallet_account, + xprv, + config.network, + &mut persister, + lookahead, + ) + .map_err(|e| { + log_error!(logger, "Failed to setup wallet for {:?}: {}", wallet_account, e); + e + })?; // Re-loaded wallets keep their persisted tip so chain backends can catch up. New wallets start // at the known tip; Esplora and Electrum recover history independently with a full scan, while @@ -1522,6 +1546,7 @@ fn build_wallet_for_account( fn build_additional_wallets( config: &Config, configured_accounts: &[OnchainWalletAccount], xprv: Xpriv, kv_store: Arc, logger: Arc, chain_tip_opt: Option<&BestBlock>, + derived_account_lookahead: u32, ) -> Result< Vec<(OnchainWalletAccount, PersistedWallet, KVStoreWalletPersister)>, BuildError, @@ -1537,6 +1562,7 @@ fn build_additional_wallets( Arc::clone(&kv_store), Arc::clone(&logger), chain_tip_opt, + None, ) { Ok(tuple) => { log_info!(logger, "Created additional wallet for {:?}", tuple.0); @@ -1561,6 +1587,7 @@ fn build_additional_wallets( Arc::clone(&kv_store), Arc::clone(&logger), chain_tip_opt, + Some(derived_account_lookahead), )?; log_info!(logger, "Loaded configured wallet for {:?}", wallet_account); additional_wallets.push(wallet); @@ -1755,6 +1782,7 @@ fn build_with_store_internal( BuildError::InvalidSeedBytes })?; let configured_accounts = validate_configured_onchain_wallet_accounts(&config, xprv)?; + let derived_account_lookahead = derived_account_lookahead(chain_data_source_config); let tx_broadcaster = Arc::new(TransactionBroadcaster::new(Arc::clone(&logger))); let fee_estimator = Arc::new(OnchainFeeEstimator::new()); @@ -1785,6 +1813,7 @@ fn build_with_store_internal( xprv, network, &mut persister, + None, ) .map(|(wallet, _loaded)| wallet); (result, persister) @@ -1951,6 +1980,7 @@ fn build_with_store_internal( Arc::clone(&kv_store), Arc::clone(&logger), chain_tip_opt.as_ref(), + derived_account_lookahead, )?; let wallet = Arc::new(Wallet::new( @@ -1966,6 +1996,7 @@ fn build_with_store_internal( Arc::clone(&address_type_runtime_config), Arc::clone(&node_metrics), Arc::clone(&logger), + derived_account_lookahead, )); // Initialize the KeysManager @@ -3291,7 +3322,8 @@ mod tests { let mut persister = KVStoreWalletPersister::new(Arc::clone(&store), Arc::clone(&logger), account); let (wallet, loaded) = - get_or_create_wallet_for_account(account, master, network, &mut persister).unwrap(); + get_or_create_wallet_for_account(account, master, network, &mut persister, None) + .unwrap(); assert!(!loaded); let descriptor = wallet.public_descriptor(KeychainKind::External).to_string(); diff --git a/src/chain/electrum.rs b/src/chain/electrum.rs index 4fca28a02..fa7989efe 100644 --- a/src/chain/electrum.rs +++ b/src/chain/electrum.rs @@ -11,7 +11,8 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bdk_chain::bdk_core::spk_client::{ FullScanRequest as BdkFullScanRequest, FullScanResponse as BdkFullScanResponse, - SyncRequest as BdkSyncRequest, SyncResponse as BdkSyncResponse, + SyncProgress as BdkSyncProgress, SyncRequest as BdkSyncRequest, + SyncResponse as BdkSyncResponse, }; use bdk_electrum::BdkElectrumClient; use bdk_wallet::{KeychainKind as BdkKeyChainKind, Update as BdkUpdate}; @@ -59,6 +60,28 @@ fn additional_wallet_full_scan_settings(config: &ElectrumSyncConfig) -> FullScan } } +fn additional_wallet_incremental_batch_size( + account_index: u32, full_scan_settings: FullScanSettings, +) -> usize { + if account_index == 0 { + BDK_ELECTRUM_CLIENT_BATCH_SIZE + } else { + full_scan_settings.batch_size + } +} + +fn derived_wallet_incremental_sync_timeout( + progress: BdkSyncProgress, batch_size: usize, +) -> Duration { + let request_count = progress + .spks_remaining + .div_ceil(batch_size) + .saturating_add(progress.txids_remaining) + .saturating_add(progress.outpoints_remaining); + let request_count = u64::try_from(request_count).unwrap_or(u64::MAX); + Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS.saturating_add(request_count)) +} + pub(super) struct ElectrumChainSource { server_url: String, pub(super) sync_config: ElectrumSyncConfig, @@ -196,7 +219,12 @@ impl ElectrumChainSource { super::WalletSyncRequest::Incremental(req) => { join_set.spawn(async move { let result: Result = client - .get_incremental_sync_wallet_update(req, txs.iter().cloned()) + .get_incremental_sync_wallet_update( + req, + txs.iter().cloned(), + BDK_ELECTRUM_CLIENT_BATCH_SIZE, + Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS), + ) .await .map(|u| u.into()); (None, result) @@ -221,11 +249,28 @@ impl ElectrumChainSource { for (wallet_account, sync_req) in additional_sync_requests { let client = Arc::clone(&electrum_client); let txs = Arc::clone(&cached_txs); + let incremental_batch_size = additional_wallet_incremental_batch_size( + wallet_account.account_index, + additional_full_scan_settings, + ); match sync_req { super::WalletSyncRequest::Incremental(req) => { + let timeout = if wallet_account.account_index == 0 { + Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS) + } else { + derived_wallet_incremental_sync_timeout( + req.progress(), + incremental_batch_size, + ) + }; join_set.spawn(async move { let result: Result = client - .get_incremental_sync_wallet_update(req, txs.iter().cloned()) + .get_incremental_sync_wallet_update( + req, + txs.iter().cloned(), + incremental_batch_size, + timeout, + ) .await .map(|u| u.into()); (Some(wallet_account), result) @@ -694,16 +739,16 @@ impl ElectrumRuntimeClient { async fn get_incremental_sync_wallet_update( &self, request: BdkSyncRequest<(BdkKeyChainKind, u32)>, - cached_txs: impl IntoIterator>>, + cached_txs: impl IntoIterator>>, batch_size: usize, + timeout: Duration, ) -> Result { let bdk_electrum_client = Arc::clone(&self.bdk_electrum_client); bdk_electrum_client.populate_tx_cache(cached_txs); - let spawn_fut = self.runtime.spawn_blocking(move || { - bdk_electrum_client.sync(request, BDK_ELECTRUM_CLIENT_BATCH_SIZE, true) - }); - let wallet_sync_timeout_fut = - tokio::time::timeout(Duration::from_secs(BDK_WALLET_SYNC_TIMEOUT_SECS), spawn_fut); + let spawn_fut = self + .runtime + .spawn_blocking(move || bdk_electrum_client.sync(request, batch_size, true)); + let wallet_sync_timeout_fut = tokio::time::timeout(timeout, spawn_fut); wallet_sync_timeout_fut .await @@ -895,4 +940,30 @@ mod tests { FullScanSettings { stop_gap: 1, batch_size: 1 } ); } + + #[test] + fn derived_incremental_sync_uses_the_configured_batch_size() { + let settings = FullScanSettings { stop_gap: 1_000, batch_size: 100 }; + + assert_eq!(additional_wallet_incremental_batch_size(0, settings), 5); + assert_eq!(additional_wallet_incremental_batch_size(1, settings), 100); + } + + #[test] + fn derived_incremental_timeout_scales_with_request_batches() { + let settings = FullScanSettings { stop_gap: 1_000, batch_size: 100 }; + let progress = BdkSyncProgress { + spks_consumed: 0, + spks_remaining: 2_000, + txids_consumed: 0, + txids_remaining: 0, + outpoints_consumed: 0, + outpoints_remaining: 0, + }; + + assert_eq!( + derived_wallet_incremental_sync_timeout(progress, settings.batch_size), + Duration::from_secs(40) + ); + } } diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 65bbf3f3f..522e04de1 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -1382,7 +1382,7 @@ mod tests { use super::{collect_additional_sync_requests, WalletSyncRequest}; use crate::builder::NodeBuilder; - use crate::config::{AddressType, Config, OnchainWalletAccount}; + use crate::config::{AddressType, Config, ElectrumSyncConfig, OnchainWalletAccount}; use crate::io::test_utils::InMemoryStore; use crate::types::DynStore; @@ -1423,4 +1423,42 @@ mod tests { .unwrap(); assert!(matches!(requests[0].1, WalletSyncRequest::Incremental(_))); } + + #[test] + fn electrum_stop_gap_sets_derived_account_incremental_lookahead() { + let mut config = Config::default(); + config.network = Network::Regtest; + let sync_config = ElectrumSyncConfig { + background_sync_config: None, + additional_wallet_full_scan_stop_gap: 1_000, + ..ElectrumSyncConfig::default() + }; + let mut builder = NodeBuilder::from_config(config); + builder.set_chain_source_electrum("ssl://127.0.0.1:1".to_string(), Some(sync_config)); + builder.set_entropy_seed_bytes([42u8; 64]); + builder.set_log_facade_logger(); + let store: Arc = Arc::new(InMemoryStore::new()); + let node = builder.build_with_store(store).unwrap(); + let account = + OnchainWalletAccount { address_type: AddressType::NativeSegwit, account_index: 1 }; + let xpub = node + .export_onchain_wallet_account_xpub(account.address_type, account.account_index) + .unwrap(); + node.add_onchain_wallet_account(account.address_type, account.account_index, xpub).unwrap(); + node.wallet.mark_derived_account_synced(account); + + let mut requests = collect_additional_sync_requests( + &[account], + &node.wallet, + &node.node_metrics, + &node.logger, + ) + .unwrap(); + match requests.pop().unwrap().1 { + WalletSyncRequest::Incremental(request) => { + assert_eq!(request.progress().spks_remaining, 1_000); + }, + WalletSyncRequest::FullScan(_) => panic!("expected incremental sync request"), + } + } } diff --git a/src/config.rs b/src/config.rs index bff05e0e1..3e1bca7cc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -848,13 +848,16 @@ pub struct ElectrumSyncConfig { pub connection_timeout_secs: u64, /// Batch size for Electrum full scans of non-primary wallets. /// - /// This applies to monitored account-`0` address types and derived accounts. It does not affect - /// the primary wallet or incremental sync. Values below `1` are treated as `1`. Defaults to `5`. + /// This applies to full scans of monitored account-`0` address types and derived accounts, and to + /// rolling incremental scans of derived accounts. It does not affect primary or account-`0` + /// incremental sync. Values below `1` are treated as `1`. Defaults to `5`. pub additional_wallet_full_scan_batch_size: u32, /// Stop gap for Electrum full scans of non-primary wallets. /// /// This applies to monitored account-`0` address types and derived accounts. It does not affect - /// the primary wallet. Values below `1` are treated as `1`. Defaults to `20`. + /// the primary wallet. For derived accounts, it also sets the incremental lookahead beyond the + /// highest revealed index. Activity within that window advances and replenishes it. Values below + /// `1` are treated as `1`. Defaults to `20`. pub additional_wallet_full_scan_stop_gap: u32, } diff --git a/src/lib.rs b/src/lib.rs index 8cf494ac4..da49f34cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2239,10 +2239,11 @@ impl Node { /// Registered funds participate in normal wallet coin selection and signing. Change remains on /// the configured primary account-`0` wallet. /// - /// Esplora/Electrum perform a full scan after registration. When issuing more addresses from the - /// exported xpub, call [`OnchainPayment::reveal_receive_addresses_to_account`] with the highest - /// issued index. Bitcoind replays the current mempool but cannot scan confirmed history from before - /// a brand-new account's first registration. + /// Esplora/Electrum perform a full scan after registration, then maintain a rolling lookahead for + /// derived accounts. Activity inside the lookahead advances and replenishes it. Use + /// [`OnchainPayment::reveal_receive_addresses_to_account`] when an external address issuer may + /// skip beyond the current lookahead. Bitcoind replays the current mempool but cannot scan + /// confirmed history from before a brand-new account's first registration. pub fn add_onchain_wallet_account( &self, address_type: AddressType, account_index: u32, xpub: String, ) -> Result<(), Error> { diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index d5388dfa1..71bbc1ca8 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -138,6 +138,7 @@ pub(crate) struct Wallet { address_type_runtime_config: Arc>, node_metrics: Arc>, logger: Arc, + derived_account_lookahead: u32, } impl Wallet { @@ -154,6 +155,7 @@ impl Wallet { config: Arc, kv_store: Arc, address_type_runtime_config: Arc>, node_metrics: Arc>, logger: Arc, + derived_account_lookahead: u32, ) -> Self { let primary_account = OnchainWalletAccount::account_zero(config.address_type); let aggregate = @@ -175,6 +177,7 @@ impl Wallet { address_type_runtime_config, node_metrics, logger, + derived_account_lookahead, } } @@ -219,7 +222,12 @@ impl Wallet { pub(crate) fn get_wallet_incremental_sync_request( &self, wallet_account: OnchainWalletAccount, ) -> Result, bdk_wallet_aggregate::Error> { - self.inner.lock().unwrap().wallet_incremental_sync_request(&wallet_account) + let aggregate = self.inner.lock().unwrap(); + if wallet_account.account_index == 0 { + aggregate.wallet_incremental_sync_request(&wallet_account) + } else { + aggregate.wallet_incremental_sync_request_with_lookahead(&wallet_account) + } } pub(crate) fn get_cached_txs(&self) -> Vec> { @@ -304,6 +312,7 @@ impl Wallet { xprv, self.config.network, &mut persister, + (account.account_index != 0).then_some(self.derived_account_lookahead), ) .map_err(|e| { log_error!(self.logger, "Failed to reload wallet {:?} after rewind: {}", account, e); @@ -430,6 +439,7 @@ impl Wallet { self.current_best_block(), Arc::clone(&self.kv_store), Arc::clone(&self.logger), + None, )?; { @@ -521,6 +531,7 @@ impl Wallet { self.current_best_block(), Arc::clone(&self.kv_store), Arc::clone(&self.logger), + None, )?) } else { None @@ -662,6 +673,7 @@ impl Wallet { self.current_best_block(), Arc::clone(&self.kv_store), Arc::clone(&self.logger), + Some(self.derived_account_lookahead), )?; { @@ -2163,7 +2175,7 @@ impl Listen for Wallet { fn create_wallet_for_account( seed_bytes: &[u8; WALLET_KEYS_SEED_LEN], network: Network, wallet_account: OnchainWalletAccount, chain_tip: BestBlock, kv_store: Arc, - logger: Arc, + logger: Arc, lookahead: Option, ) -> Result<(PersistedWallet, KVStoreWalletPersister), Error> { let xprv = Xpriv::new_master(network, seed_bytes).map_err(|e| { log_error!(logger, "Failed to derive master secret: {}", e); @@ -2178,6 +2190,7 @@ fn create_wallet_for_account( xprv, network, &mut persister, + lookahead, ) .map_err(|e| { log_error!(logger, "Failed to setup wallet for {:?}: {}", wallet_account, e); diff --git a/tests/multi_address_types_tests.rs b/tests/multi_address_types_tests.rs index 08b21e5b5..3dad34b7f 100644 --- a/tests/multi_address_types_tests.rs +++ b/tests/multi_address_types_tests.rs @@ -3926,6 +3926,75 @@ mod derived_accounts { .total_sats, 100_000 ); + node.stop().unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn test_electrum_derived_account_incremental_sync_rolls_lookahead() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + premine_blocks(&bitcoind.client, &electrsd.client).await; + + let config = node_config(AddressType::NativeSegwit, vec![]); + crate::common::setup_builder!(builder, config.node_config); + let electrum_url = format!("tcp://{}", electrsd.electrum_url); + let sync_config = ElectrumSyncConfig { + background_sync_config: None, + additional_wallet_full_scan_batch_size: 100, + additional_wallet_full_scan_stop_gap: 1_000, + ..Default::default() + }; + builder.set_chain_source_electrum(electrum_url, Some(sync_config)); + builder.set_log_facade_logger(); + let node = builder.build().unwrap(); + node.start().unwrap(); + + let xpub = node.export_onchain_wallet_account_xpub(AddressType::NativeSegwit, 1).unwrap(); + node.add_onchain_wallet_account(AddressType::NativeSegwit, 1, xpub.clone()).unwrap(); + node.onchain_payment() + .reveal_receive_addresses_to_account(AddressType::NativeSegwit, 1, 999) + .unwrap(); + node.sync_wallets().unwrap(); + + let index_990 = native_segwit_receive_address_from_xpub(&xpub, 990); + distribute_funds_unconfirmed( + &bitcoind.client, + &electrsd.client, + vec![index_990], + Amount::from_sat(100_000), + ) + .await; + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 1).await; + node.sync_wallets().unwrap(); + assert_eq!( + node.get_balance_for_onchain_wallet_account(AddressType::NativeSegwit, 1) + .unwrap() + .total_sats, + 100_000 + ); + + let index_1200 = native_segwit_receive_address_from_xpub(&xpub, 1_200); + distribute_funds_unconfirmed( + &bitcoind.client, + &electrsd.client, + vec![index_1200], + Amount::from_sat(70_000), + ) + .await; + generate_blocks_and_wait(&bitcoind.client, &electrsd.client, 1).await; + node.sync_wallets().unwrap(); + assert_eq!( + node.get_balance_for_onchain_wallet_account(AddressType::NativeSegwit, 1) + .unwrap() + .total_sats, + 170_000 + ); + assert_eq!( + node.onchain_payment() + .new_address_info_for_account(AddressType::NativeSegwit, 1) + .unwrap() + .index, + 1_201 + ); node.stop().unwrap(); }