zcb::sync: Refresh UTXOs at the start of each scanning cycle

zcash_client_backend/CHANGELOG.md

@@ -11,6 +11,8 @@ and this library adheres to Rust's notion of
 - The `Account` trait now uses an associated type for its `AccountId`
   type instead of a type parameter. This change allows for the simplification
   of some type signatures.
+- `zcash_client_backend::sync::run`:
+  - Transparent outputs are now refreshed in addition to shielded notes.
 
 ### Fixed
 - `zcash_client_backend::tor::grpc` now needs the `lightwalletd-tonic-tls-webpki-roots`

zcash_client_backend/src/sync.rs

@@ -41,6 +41,16 @@
 #[cfg(feature = "orchard")]
 use orchard::tree::MerkleHashOrchard;
 
+#[cfg(feature = "transparent-inputs")]
+use {
+    crate::{encoding::AddressCodec, wallet::WalletTransparentOutput},
+    zcash_primitives::{
+        legacy::Script,
+        transaction::components::transparent::{OutPoint, TxOut},
+    },
+    zcash_protocol::{consensus::NetworkUpgrade, value::Zatoshis},
+};
+
 /// Scans the chain until the wallet is up-to-date.
 pub async fn run<P, ChT, CaT, DbT>(
     client: &mut CompactTxStreamerClient<ChT>,
@@ -62,11 +72,27 @@
     <DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
     <DbT as WalletCommitmentTrees>::Error: std::error::Error + Send + Sync + 'static,
 {
+    #[cfg(feature = "transparent-inputs")]
+    let wallet_birthday = db_data
+        .get_wallet_birthday()
+        .map_err(Error::Wallet)?
+        .unwrap_or_else(|| params.activation_height(NetworkUpgrade::Sapling).unwrap());
+
     // 1) Download note commitment tree data from lightwalletd
     // 2) Pass the commitment tree data to the database.
     update_subtree_roots(client, db_data).await?;
 
-    while running(client, params, db_cache, db_data, batch_size).await? {}
+    while running(
+        client,
+        params,
+        db_cache,
+        db_data,
+        batch_size,
+        #[cfg(feature = "transparent-inputs")]
+        wallet_birthday,
+    )
+    .await?
+    {}
 
     Ok(())
 }
@@ -77,6 +103,7 @@
     db_cache: &CaT,
     db_data: &mut DbT,
     batch_size: u32,
+    #[cfg(feature = "transparent-inputs")] wallet_birthday: BlockHeight,
 ) -> Result<bool, Error<CaT::Error, <DbT as WalletRead>::Error, TrErr>>
 where
     P: Parameters + Send + 'static,
@@ -94,6 +121,23 @@
     // 4) Notify the wallet of the updated chain tip.
     update_chain_tip(client, db_data).await?;
 
+    // Refresh UTXOs for the accounts in the wallet. We do this before we perform
+    // any shielded scanning, to ensure that we discover any UTXOs between the old
+    // fully-scanned height and the current chain tip.
+    #[cfg(feature = "transparent-inputs")]
+    for account_id in db_data.get_account_ids().map_err(Error::Wallet)? {
+        let start_height = db_data
+            .block_fully_scanned()
+            .map_err(Error::Wallet)?
+            .map(|meta| meta.block_height())
+            .unwrap_or(wallet_birthday);
+        info!(
+            "Refreshing UTXOs for {:?} from height {}",
+            account_id, start_height,
+        );
+        refresh_utxos(params, client, db_data, account_id, start_height).await?;
+    }
+
     // 5) Get the suggested scan ranges from the wallet database
     let mut scan_ranges = db_data.suggest_scan_ranges().map_err(Error::Wallet)?;
 
@@ -422,6 +466,100 @@
     }
 }
 
+/// Refreshes the given account's view of UTXOs that exist starting at the given height.
+///
+/// ## Note about UTXO tracking
+///
+/// (Extracted from [a comment in the Android SDK].)
+///
+/// We no longer clear UTXOs here, as `WalletDb::put_received_transparent_utxo` now uses
+/// an upsert instead of an insert. This means that now-spent UTXOs would previously have
+/// been deleted, but now are left in the database (like shielded notes).
+///
+/// Due to the fact that the `lightwalletd` query only returns _current_ UTXOs, we don't
+/// learn about recently-spent UTXOs here, so the transparent balance does not get updated
+/// here.
+///
+/// Instead, when a received shielded note is "enhanced" by downloading the full
+/// transaction, we mark any UTXOs spent in that transaction as spent in the database.
+/// This relies on two current properties:
+/// - UTXOs are only ever spent in shielding transactions.
+/// - At least one shielded note from each shielding transaction is always enhanced.
+///
+/// However, for greater reliability, we may want to alter the Data Access API to support
+/// "inferring spentness" from what is _not_ returned as a UTXO, or alternatively fetch
+/// TXOs from `lightwalletd` instead of just UTXOs.
+///
+/// [a comment in the Android SDK]: https://github.com/Electric-Coin-Company/zcash-android-wallet-sdk/blob/855204fc8ae4057fdac939f98df4aa38c8e662f1/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/processor/CompactBlockProcessor.kt#L979-L991
+#[cfg(feature = "transparent-inputs")]
+async fn refresh_utxos<P, ChT, DbT, CaErr, TrErr>(
+    params: &P,
+    client: &mut CompactTxStreamerClient<ChT>,
+    db_data: &mut DbT,
+    account_id: DbT::AccountId,
+    start_height: BlockHeight,
+) -> Result<(), Error<CaErr, <DbT as WalletRead>::Error, TrErr>>
+where
+    P: Parameters + Send + 'static,
+    ChT: GrpcService<BoxBody>,
+    ChT::Error: Into<StdError>,
+    ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
+    <ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
+    DbT: WalletWrite,
+    DbT::Error: std::error::Error + Send + Sync + 'static,
+{
+    let request = service::GetAddressUtxosArg {
+        addresses: db_data
+            .get_transparent_receivers(account_id)
+            .map_err(Error::Wallet)?
+            .into_keys()
+            .map(|addr| addr.encode(params))
+            .collect(),
+        start_height: start_height.into(),
+        max_entries: 0,
+    };
+
+    if request.addresses.is_empty() {
+        info!("{:?} has no transparent receivers", account_id);
+    } else {
+        client
+            .get_address_utxos_stream(request)
+            .await?
+            .into_inner()
+            .map_err(Error::Server)
+            .and_then(|reply| async move {
+                WalletTransparentOutput::from_parts(
+                    OutPoint::new(
+                        reply.txid[..]
+                            .try_into()
+                            .map_err(|_| Error::MisbehavingServer)?,
+                        reply
+                            .index
+                            .try_into()
+                            .map_err(|_| Error::MisbehavingServer)?,
+                    ),
+                    TxOut {
+                        value: Zatoshis::from_nonnegative_i64(reply.value_zat)
+                            .map_err(|_| Error::MisbehavingServer)?,
+                        script_pubkey: Script(reply.script),
+                    },
+                    Some(
+                        BlockHeight::try_from(reply.height)
+                            .map_err(|_| Error::MisbehavingServer)?,
+                    ),
+                )
+                .ok_or(Error::MisbehavingServer)
+            })
+            .try_for_each(|output| {
+                let res = db_data.put_received_transparent_utxo(&output).map(|_| ());
+                async move { res.map_err(Error::Wallet) }
+            })
+            .await?;
+    }
+
+    Ok(())
+}
+
 /// Errors that can occur while syncing.
 #[derive(Debug)]
 pub enum Error<CaErr, DbErr, TrErr> {