From 861d829420ff28e387642659ffe256955352d939 Mon Sep 17 00:00:00 2001 From: Benjamin Kampmann Date: Mon, 11 Jun 2018 20:38:01 +0200 Subject: [PATCH 1/5] Fix Cli Return Code on --help for ethkey, ethstore & whisper (#8863) Docopt handles `--help` automatically for us, however we've handled those Errors the same as all others: by exiting with Return Code `1`, which is wrong for a totally appropriate a quit on `--help`. Fortunately `docopt:Error` provides an `exit` helper function that discriminates properly between fatal and non-fatal errors and exist appropriately. This patch makes sure we use that handy function in case we encounter such an error in the CLI of ethkey, ethstore and whisper. Thus those are now giving the appropriate Return code on `--help`. fixes #8851 --- ethkey/cli/src/main.rs | 3 ++- ethstore/cli/src/main.rs | 1 + whisper/cli/src/main.rs | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ethkey/cli/src/main.rs b/ethkey/cli/src/main.rs index c8f5e2e64e7..af50f86b31a 100644 --- a/ethkey/cli/src/main.rs +++ b/ethkey/cli/src/main.rs @@ -166,10 +166,11 @@ fn main() { match execute(env::args()) { Ok(ok) => println!("{}", ok), + Err(Error::Docopt(ref e)) => e.exit(), Err(err) => { println!("{}", err); process::exit(1); - }, + } } } diff --git a/ethstore/cli/src/main.rs b/ethstore/cli/src/main.rs index 416b64d43e0..922d857149d 100644 --- a/ethstore/cli/src/main.rs +++ b/ethstore/cli/src/main.rs @@ -149,6 +149,7 @@ fn main() { match execute(env::args()) { Ok(result) => println!("{}", result), + Err(Error::Docopt(ref e)) => e.exit(), Err(err) => { println!("{}", err); process::exit(1); diff --git a/whisper/cli/src/main.rs b/whisper/cli/src/main.rs index 6f3aec8594e..1d57691418b 100644 --- a/whisper/cli/src/main.rs +++ b/whisper/cli/src/main.rs @@ -190,11 +190,12 @@ fn main() { Ok(_) => { println!("whisper-cli terminated"); process::exit(1); - } + }, + Err(Error::Docopt(ref e)) => e.exit(), Err(err) => { println!("{}", err); process::exit(1); - }, + } } } From 2a470deeafa194dbb53d277d7839cd8c78deed28 Mon Sep 17 00:00:00 2001 From: Andronik Ordian Date: Tue, 12 Jun 2018 09:15:52 +0300 Subject: [PATCH 2/5] Don't allocate in expect_valid_rlp unless necessary (#8867) * don't allocate via format! in case there's no error * fix test? --- ethcore/src/views/view_rlp.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ethcore/src/views/view_rlp.rs b/ethcore/src/views/view_rlp.rs index 2ecc4dbdd3d..2dd1b33a946 100644 --- a/ethcore/src/views/view_rlp.rs +++ b/ethcore/src/views/view_rlp.rs @@ -39,10 +39,10 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view { /// Returns a new instance replacing existing rlp with new rlp, maintaining debug info fn new_from_rlp(&self, rlp: Rlp<'a>) -> Self { - ViewRlp { - rlp, + ViewRlp { + rlp, file: self.file, - line: self.line + line: self.line } } @@ -53,7 +53,12 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view { } fn expect_valid_rlp(&self, r: Result) -> T { - r.expect(&format!("View rlp is trusted and should be valid. Constructed in {} on line {}", self.file, self.line)) + r.unwrap_or_else(|e| panic!( + "View rlp is trusted and should be valid. Constructed in {} on line {}: {}", + self.file, + self.line, + e + )) } /// Returns rlp at the given index, panics if no rlp at that index @@ -75,7 +80,7 @@ impl<'a, 'view> ViewRlp<'a> where 'a : 'view { /// Returns decoded value at the given index, panics not present or valid at that index pub fn val_at(&self, index: usize) -> T where T : Decodable { self.expect_valid_rlp(self.rlp.val_at(index)) - } + } /// Returns decoded list of values, panics if rlp is invalid pub fn list_at(&self, index: usize) -> Vec where T: Decodable { From 4817b94d0bb23356a5ee9b659ab8f4c4cbbd660d Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 12 Jun 2018 14:21:55 +0800 Subject: [PATCH 3/5] Use sealing.enabled to emit eth_mining information (#8844) * Use sealing.enabled to emit eth_mining information * Be more accurate on last_requests by using Option * Add tests for internal sealing * Add test for pow non-mining * Add test for mining pow --- ethcore/src/miner/miner.rs | 71 +++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index bf51d0b135b..e91e03e8bc7 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -180,7 +180,7 @@ struct SealingWork { next_allowed_reseal: Instant, next_mandatory_reseal: Instant, // block number when sealing work was last requested - last_request: u64, + last_request: Option, } impl SealingWork { @@ -231,7 +231,7 @@ impl Miner { || spec.engine.seals_internally().is_some(), next_allowed_reseal: Instant::now(), next_mandatory_reseal: Instant::now() + options.reseal_max_period, - last_request: 0, + last_request: None, }), params: RwLock::new(AuthoringParams::default()), listeners: RwLock::new(vec![]), @@ -496,8 +496,10 @@ impl Miner { trace!(target: "miner", "requires_reseal: sealing enabled"); // Disable sealing if there were no requests for SEALING_TIMEOUT_IN_BLOCKS - let had_requests = best_block > sealing.last_request - && best_block - sealing.last_request <= SEALING_TIMEOUT_IN_BLOCKS; + let had_requests = sealing.last_request.map(|last_request| { + best_block > last_request + && best_block - last_request <= SEALING_TIMEOUT_IN_BLOCKS + }).unwrap_or(false); // keep sealing enabled if any of the conditions is met let sealing_enabled = self.forced_sealing() @@ -516,7 +518,7 @@ impl Miner { ); if should_disable_sealing { - trace!(target: "miner", "Miner sleeping (current {}, last {})", best_block, sealing.last_request); + trace!(target: "miner", "Miner sleeping (current {}, last {})", best_block, sealing.last_request.unwrap_or(0)); sealing.enabled = false; sealing.queue.reset(); false @@ -676,13 +678,13 @@ impl Miner { let best_number = client.chain_info().best_block_number; let mut sealing = self.sealing.lock(); - if sealing.last_request != best_number { + if sealing.last_request != Some(best_number) { trace!( target: "miner", "prepare_pending_block: Miner received request (was {}, now {}) - waking up.", - sealing.last_request, best_number + sealing.last_request.unwrap_or(0), best_number ); - sealing.last_request = best_number; + sealing.last_request = Some(best_number); } // Return if we restarted @@ -954,7 +956,7 @@ impl miner::MinerService for Miner { } fn is_currently_sealing(&self) -> bool { - self.sealing.lock().queue.is_in_use() + self.sealing.lock().enabled } fn work_package(&self, chain: &C) -> Option<(H256, BlockNumber, u64, U256)> where @@ -1266,4 +1268,55 @@ mod tests { let client = generate_dummy_client_with_spec_and_accounts(spec, None); assert!(match client.miner().set_author(addr, Some("".into())) { Err(AccountError::NotFound) => true, _ => false }); } + + #[test] + fn should_mine_if_internal_sealing_is_enabled() { + let spec = Spec::new_instant(); + let miner = Miner::new_for_tests(&spec, None); + + let client = generate_dummy_client(2); + miner.update_sealing(&*client); + + assert!(miner.is_currently_sealing()); + } + + #[test] + fn should_not_mine_if_internal_sealing_is_disabled() { + let spec = Spec::new_test_round(); + let miner = Miner::new_for_tests(&spec, None); + + let client = generate_dummy_client(2); + miner.update_sealing(&*client); + + assert!(!miner.is_currently_sealing()); + } + + #[test] + fn should_not_mine_if_no_fetch_work_request() { + let spec = Spec::new_test(); + let miner = Miner::new_for_tests(&spec, None); + + let client = generate_dummy_client(2); + miner.update_sealing(&*client); + + assert!(!miner.is_currently_sealing()); + } + + #[test] + fn should_mine_if_fetch_work_request() { + struct DummyNotifyWork; + + impl NotifyWork for DummyNotifyWork { + fn notify(&self, _pow_hash: H256, _difficulty: U256, _number: u64) { } + } + + let spec = Spec::new_test(); + let miner = Miner::new_for_tests(&spec, None); + miner.add_work_listener(Box::new(DummyNotifyWork)); + + let client = generate_dummy_client(2); + miner.update_sealing(&*client); + + assert!(miner.is_currently_sealing()); + } } From 4938d5dde5c83403e6cffc7301ebd35cbdcf04a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 12 Jun 2018 08:22:54 +0200 Subject: [PATCH 4/5] Limit the number of transactions in pending set (#8777) * Unordered iterator. * Use unordered and limited set if full not required. * Split timeout work into smaller timers. * Avoid collecting all pending transactions when mining * Remove println. * Use priority ordering in eth-filter. * Fix ethcore-miner tests and tx propagation. * Review grumbles addressed. * Add test for unordered not populating the cache. * Fix ethcore tests. * Fix light tests. * Fix ethcore-sync tests. * Fix RPC tests. --- Cargo.lock | 6 +- ethcore/light/src/net/mod.rs | 5 +- ethcore/light/src/net/tests/mod.rs | 4 +- ethcore/light/src/provider.rs | 13 ++- ethcore/src/client/client.rs | 4 +- ethcore/src/client/test_client.rs | 6 +- ethcore/src/client/traits.rs | 2 +- ethcore/src/miner/miner.rs | 51 ++++++--- ethcore/src/miner/mod.rs | 7 +- ethcore/src/tests/client.rs | 10 +- ethcore/sync/src/api.rs | 21 +++- ethcore/sync/src/chain/mod.rs | 10 +- ethcore/sync/src/chain/propagator.rs | 3 +- miner/src/lib.rs | 3 +- miner/src/pool/mod.rs | 39 ++++++- miner/src/pool/queue.rs | 71 ++++++++++--- miner/src/pool/tests/mod.rs | 124 ++++++++++++++++------ parity/rpc_apis.rs | 4 +- rpc/Cargo.toml | 2 +- rpc/src/v1/impls/eth_filter.rs | 2 +- rpc/src/v1/impls/eth_pubsub.rs | 2 +- rpc/src/v1/impls/light/parity.rs | 3 +- rpc/src/v1/impls/parity.rs | 10 +- rpc/src/v1/tests/helpers/miner_service.rs | 4 +- rpc/src/v1/tests/mocked/eth_pubsub.rs | 2 +- rpc/src/v1/traits/parity.rs | 2 +- transaction-pool/Cargo.toml | 2 +- transaction-pool/src/pool.rs | 58 +++++++++- transaction-pool/src/tests/mod.rs | 60 +++++++++++ 29 files changed, 415 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9334d01028d..60565f17585 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,7 +694,7 @@ dependencies = [ "rlp 0.2.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.0", - "transaction-pool 1.12.0", + "transaction-pool 1.12.1", "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2241,7 +2241,7 @@ dependencies = [ "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "transaction-pool 1.12.0", + "transaction-pool 1.12.1", "transient-hashmap 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", ] @@ -3422,7 +3422,7 @@ dependencies = [ [[package]] name = "transaction-pool" -version = "1.12.0" +version = "1.12.1" dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 39f53445ada..179f7090036 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -72,6 +72,9 @@ const PROPAGATE_TIMEOUT_INTERVAL: Duration = Duration::from_secs(5); const RECALCULATE_COSTS_TIMEOUT: TimerToken = 3; const RECALCULATE_COSTS_INTERVAL: Duration = Duration::from_secs(60 * 60); +/// Max number of transactions in a single packet. +const MAX_TRANSACTIONS_TO_PROPAGATE: usize = 64; + // minimum interval between updates. const UPDATE_INTERVAL: Duration = Duration::from_millis(5000); @@ -647,7 +650,7 @@ impl LightProtocol { fn propagate_transactions(&self, io: &IoContext) { if self.capabilities.read().tx_relay { return } - let ready_transactions = self.provider.ready_transactions(); + let ready_transactions = self.provider.ready_transactions(MAX_TRANSACTIONS_TO_PROPAGATE); if ready_transactions.is_empty() { return } trace!(target: "pip", "propagate transactions: {} ready", ready_transactions.len()); diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index 305ef9b3540..e182f38b8a6 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -173,8 +173,8 @@ impl Provider for TestProvider { }) } - fn ready_transactions(&self) -> Vec { - self.0.client.ready_transactions() + fn ready_transactions(&self, max_len: usize) -> Vec { + self.0.client.ready_transactions(max_len) } } diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 0e518ea7723..e75915e8cf8 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -125,7 +125,7 @@ pub trait Provider: Send + Sync { fn header_proof(&self, req: request::CompleteHeaderProofRequest) -> Option; /// Provide pending transactions. - fn ready_transactions(&self) -> Vec; + fn ready_transactions(&self, max_len: usize) -> Vec; /// Provide a proof-of-execution for the given transaction proof request. /// Returns a vector of all state items necessary to execute the transaction. @@ -280,8 +280,8 @@ impl Provider for T { .map(|(_, proof)| ::request::ExecutionResponse { items: proof }) } - fn ready_transactions(&self) -> Vec { - BlockChainClient::ready_transactions(self) + fn ready_transactions(&self, max_len: usize) -> Vec { + BlockChainClient::ready_transactions(self, max_len) .into_iter() .map(|tx| tx.pending().clone()) .collect() @@ -367,9 +367,12 @@ impl Provider for LightProvider { None } - fn ready_transactions(&self) -> Vec { + fn ready_transactions(&self, max_len: usize) -> Vec { let chain_info = self.chain_info(); - self.txqueue.read().ready_transactions(chain_info.best_block_number, chain_info.best_block_timestamp) + let mut transactions = self.txqueue.read() + .ready_transactions(chain_info.best_block_number, chain_info.best_block_timestamp); + transactions.truncate(max_len); + transactions } } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 2d0201067f1..d47d1afa49b 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -1956,8 +1956,8 @@ impl BlockChainClient for Client { (*self.build_last_hashes(&self.chain.read().best_block_hash())).clone() } - fn ready_transactions(&self) -> Vec> { - self.importer.miner.ready_transactions(self) + fn ready_transactions(&self, max_len: usize) -> Vec> { + self.importer.miner.ready_transactions(self, max_len, ::miner::PendingOrdering::Priority) } fn signing_chain_id(&self) -> Option { diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 4bd76b4495c..620f1af3330 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -48,7 +48,7 @@ use log_entry::LocalizedLogEntry; use receipt::{Receipt, LocalizedReceipt, TransactionOutcome}; use error::ImportResult; use vm::Schedule; -use miner::{Miner, MinerService}; +use miner::{self, Miner, MinerService}; use spec::Spec; use types::basic_account::BasicAccount; use types::pruning_info::PruningInfo; @@ -806,8 +806,8 @@ impl BlockChainClient for TestBlockChainClient { self.traces.read().clone() } - fn ready_transactions(&self) -> Vec> { - self.miner.ready_transactions(self) + fn ready_transactions(&self, max_len: usize) -> Vec> { + self.miner.ready_transactions(self, max_len, miner::PendingOrdering::Priority) } fn signing_chain_id(&self) -> Option { None } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index d3a20dcff89..29876fe80a4 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -321,7 +321,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra fn last_hashes(&self) -> LastHashes; /// List all transactions that are allowed into the next block. - fn ready_transactions(&self) -> Vec>; + fn ready_transactions(&self, max_len: usize) -> Vec>; /// Sorted list of transaction gas prices from at least last sample_size blocks. fn gas_price_corpus(&self, sample_size: usize) -> ::stats::Corpus { diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index e91e03e8bc7..2547fea62bf 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -364,18 +364,28 @@ impl Miner { let client = self.pool_client(chain); let engine_params = self.engine.params(); - let min_tx_gas = self.engine.schedule(chain_info.best_block_number).tx_gas.into(); + let min_tx_gas: U256 = self.engine.schedule(chain_info.best_block_number).tx_gas.into(); let nonce_cap: Option = if chain_info.best_block_number + 1 >= engine_params.dust_protection_transition { Some((engine_params.nonce_cap_increment * (chain_info.best_block_number + 1)).into()) } else { None }; + // we will never need more transactions than limit divided by min gas + let max_transactions = if min_tx_gas.is_zero() { + usize::max_value() + } else { + (*open_block.block().header().gas_limit() / min_tx_gas).as_u64() as usize + }; let pending: Vec> = self.transaction_queue.pending( client.clone(), - chain_info.best_block_number, - chain_info.best_block_timestamp, - nonce_cap, + pool::PendingSettings { + block_number: chain_info.best_block_number, + current_timestamp: chain_info.best_block_timestamp, + nonce_cap, + max_len: max_transactions, + ordering: miner::PendingOrdering::Priority, + } ); let took_ms = |elapsed: &Duration| { @@ -807,20 +817,28 @@ impl miner::MinerService for Miner { self.transaction_queue.all_transactions() } - fn ready_transactions(&self, chain: &C) -> Vec> where + fn ready_transactions(&self, chain: &C, max_len: usize, ordering: miner::PendingOrdering) + -> Vec> + where C: ChainInfo + Nonce + Sync, { let chain_info = chain.chain_info(); let from_queue = || { + // We propagate transactions over the nonce cap. + // The mechanism is only to limit number of transactions in pending block + // those transactions are valid and will just be ready to be included in next block. + let nonce_cap = None; + self.transaction_queue.pending( CachedNonceClient::new(chain, &self.nonce_cache), - chain_info.best_block_number, - chain_info.best_block_timestamp, - // We propagate transactions over the nonce cap. - // The mechanism is only to limit number of transactions in pending block - // those transactions are valid and will just be ready to be included in next block. - None, + pool::PendingSettings { + block_number: chain_info.best_block_number, + current_timestamp: chain_info.best_block_timestamp, + nonce_cap, + max_len, + ordering, + }, ) }; @@ -830,6 +848,7 @@ impl miner::MinerService for Miner { .iter() .map(|signed| pool::VerifiedTransaction::from_pending_block_transaction(signed.clone())) .map(Arc::new) + .take(max_len) .collect() }, chain_info.best_block_number) }; @@ -1083,7 +1102,7 @@ mod tests { use rustc_hex::FromHex; use client::{TestBlockChainClient, EachBlockWith, ChainInfo, ImportSealedBlock}; - use miner::MinerService; + use miner::{MinerService, PendingOrdering}; use test_helpers::{generate_dummy_client, generate_dummy_client_with_spec_and_accounts}; use transaction::{Transaction}; @@ -1179,7 +1198,7 @@ mod tests { assert_eq!(res.unwrap(), ()); assert_eq!(miner.pending_transactions(best_block).unwrap().len(), 1); assert_eq!(miner.pending_receipts(best_block).unwrap().len(), 1); - assert_eq!(miner.ready_transactions(&client).len(), 1); + assert_eq!(miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 1); // This method will let us know if pending block was created (before calling that method) assert!(!miner.prepare_pending_block(&client)); } @@ -1198,7 +1217,7 @@ mod tests { assert_eq!(res.unwrap(), ()); assert_eq!(miner.pending_transactions(best_block), None); assert_eq!(miner.pending_receipts(best_block), None); - assert_eq!(miner.ready_transactions(&client).len(), 1); + assert_eq!(miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 1); } #[test] @@ -1217,11 +1236,11 @@ mod tests { assert_eq!(miner.pending_transactions(best_block), None); assert_eq!(miner.pending_receipts(best_block), None); // By default we use PendingSet::AlwaysSealing, so no transactions yet. - assert_eq!(miner.ready_transactions(&client).len(), 0); + assert_eq!(miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 0); // This method will let us know if pending block was created (before calling that method) assert!(miner.prepare_pending_block(&client)); // After pending block is created we should see a transaction. - assert_eq!(miner.ready_transactions(&client).len(), 1); + assert_eq!(miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 1); } #[test] diff --git a/ethcore/src/miner/mod.rs b/ethcore/src/miner/mod.rs index dd5f28feb60..631941de611 100644 --- a/ethcore/src/miner/mod.rs +++ b/ethcore/src/miner/mod.rs @@ -26,6 +26,7 @@ pub mod pool_client; pub mod stratum; pub use self::miner::{Miner, MinerOptions, Penalization, PendingSet, AuthoringParams}; +pub use ethcore_miner::pool::PendingOrdering; use std::sync::Arc; use std::collections::BTreeMap; @@ -156,10 +157,12 @@ pub trait MinerService : Send + Sync { fn next_nonce(&self, chain: &C, address: &Address) -> U256 where C: Nonce + Sync; - /// Get a list of all ready transactions. + /// Get a list of all ready transactions either ordered by priority or unordered (cheaper). /// /// Depending on the settings may look in transaction pool or only in pending block. - fn ready_transactions(&self, chain: &C) -> Vec> + /// If you don't need a full set of transactions, you can add `max_len` and create only a limited set of + /// transactions. + fn ready_transactions(&self, chain: &C, max_len: usize, ordering: PendingOrdering) -> Vec> where C: ChainInfo + Nonce + Sync; /// Get a list of all transactions in the pool (some of them might not be ready for inclusion yet). diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index ccafcf6613c..e18b4db9832 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -30,7 +30,7 @@ use test_helpers::{ use types::filter::Filter; use ethereum_types::{U256, Address}; use kvdb_rocksdb::{Database, DatabaseConfig}; -use miner::Miner; +use miner::{Miner, PendingOrdering}; use spec::Spec; use views::BlockView; use ethkey::KeyPair; @@ -343,12 +343,12 @@ fn does_not_propagate_delayed_transactions() { client.miner().import_own_transaction(&*client, tx0).unwrap(); client.miner().import_own_transaction(&*client, tx1).unwrap(); - assert_eq!(0, client.ready_transactions().len()); - assert_eq!(0, client.miner().ready_transactions(&*client).len()); + assert_eq!(0, client.ready_transactions(10).len()); + assert_eq!(0, client.miner().ready_transactions(&*client, 10, PendingOrdering::Priority).len()); push_blocks_to_client(&client, 53, 2, 2); client.flush_queue(); - assert_eq!(2, client.ready_transactions().len()); - assert_eq!(2, client.miner().ready_transactions(&*client).len()); + assert_eq!(2, client.ready_transactions(10).len()); + assert_eq!(2, client.miner().ready_transactions(&*client, 10, PendingOrdering::Priority).len()); } #[test] diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index b759fb734a7..56bc579ad88 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -359,6 +359,10 @@ impl SyncProvider for EthSync { } } +const PEERS_TIMER: TimerToken = 0; +const SYNC_TIMER: TimerToken = 1; +const TX_TIMER: TimerToken = 2; + struct SyncProtocolHandler { /// Shared blockchain client. chain: Arc, @@ -373,7 +377,9 @@ struct SyncProtocolHandler { impl NetworkProtocolHandler for SyncProtocolHandler { fn initialize(&self, io: &NetworkContext) { if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID { - io.register_timer(0, Duration::from_secs(1)).expect("Error registering sync timer"); + io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer"); + io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer"); } } @@ -399,12 +405,17 @@ impl NetworkProtocolHandler for SyncProtocolHandler { } } - fn timeout(&self, io: &NetworkContext, _timer: TimerToken) { + fn timeout(&self, io: &NetworkContext, timer: TimerToken) { trace_time!("sync::timeout"); let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay); - self.sync.write().maintain_peers(&mut io); - self.sync.write().maintain_sync(&mut io); - self.sync.write().propagate_new_transactions(&mut io); + match timer { + PEERS_TIMER => self.sync.write().maintain_peers(&mut io), + SYNC_TIMER => self.sync.write().maintain_sync(&mut io), + TX_TIMER => { + self.sync.write().propagate_new_transactions(&mut io); + }, + _ => warn!("Unknown timer {} triggered.", timer), + } } } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 8f0aff7514b..84e6344e688 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -149,6 +149,10 @@ const MAX_NEW_HASHES: usize = 64; const MAX_NEW_BLOCK_AGE: BlockNumber = 20; // maximal packet size with transactions (cannot be greater than 16MB - protocol limitation). const MAX_TRANSACTION_PACKET_SIZE: usize = 8 * 1024 * 1024; +// Maximal number of transactions queried from miner to propagate. +// This set is used to diff with transactions known by the peer and +// we will send a difference of length up to `MAX_TRANSACTIONS_TO_PROPAGATE`. +const MAX_TRANSACTIONS_TO_QUERY: usize = 4096; // Maximal number of transactions in sent in single packet. const MAX_TRANSACTIONS_TO_PROPAGATE: usize = 64; // Min number of blocks to be behind for a snapshot sync @@ -1143,7 +1147,7 @@ pub mod tests { use super::{PeerInfo, PeerAsking}; use ethcore::header::*; use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient, ChainInfo, BlockInfo}; - use ethcore::miner::MinerService; + use ethcore::miner::{MinerService, PendingOrdering}; use private_tx::NoopPrivateTxHandler; pub fn get_dummy_block(order: u32, parent_hash: H256) -> Bytes { @@ -1355,7 +1359,7 @@ pub mod tests { let mut io = TestIo::new(&mut client, &ss, &queue, None); io.chain.miner.chain_new_blocks(io.chain, &[], &[], &[], &good_blocks, false); sync.chain_new_blocks(&mut io, &[], &[], &[], &good_blocks, &[], &[]); - assert_eq!(io.chain.miner.ready_transactions(io.chain).len(), 1); + assert_eq!(io.chain.miner.ready_transactions(io.chain, 10, PendingOrdering::Priority).len(), 1); } // We need to update nonce status (because we say that the block has been imported) for h in &[good_blocks[0]] { @@ -1371,7 +1375,7 @@ pub mod tests { } // then - assert_eq!(client.miner.ready_transactions(&client).len(), 1); + assert_eq!(client.miner.ready_transactions(&client, 10, PendingOrdering::Priority).len(), 1); } #[test] diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 4ae0518a537..75cf550f28b 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -33,6 +33,7 @@ use super::{ MAX_PEERS_PROPAGATION, MAX_TRANSACTION_PACKET_SIZE, MAX_TRANSACTIONS_TO_PROPAGATE, + MAX_TRANSACTIONS_TO_QUERY, MIN_PEERS_PROPAGATION, CONSENSUS_DATA_PACKET, NEW_BLOCK_HASHES_PACKET, @@ -114,7 +115,7 @@ impl SyncPropagator { return 0; } - let transactions = io.chain().ready_transactions(); + let transactions = io.chain().ready_transactions(MAX_TRANSACTIONS_TO_QUERY); if transactions.is_empty() { return 0; } diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 9dc180fefe4..6b61df4de4e 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -30,13 +30,14 @@ extern crate linked_hash_map; extern crate parking_lot; extern crate price_info; extern crate rlp; -extern crate trace_time; extern crate transaction_pool as txpool; #[macro_use] extern crate error_chain; #[macro_use] extern crate log; +#[macro_use] +extern crate trace_time; #[cfg(test)] extern crate rustc_hex; diff --git a/miner/src/pool/mod.rs b/miner/src/pool/mod.rs index 57f813157bd..fd4ef6ef2e8 100644 --- a/miner/src/pool/mod.rs +++ b/miner/src/pool/mod.rs @@ -16,7 +16,7 @@ //! Transaction Pool -use ethereum_types::{H256, Address}; +use ethereum_types::{U256, H256, Address}; use heapsize::HeapSizeOf; use transaction; use txpool; @@ -45,6 +45,43 @@ pub enum PrioritizationStrategy { GasPriceOnly, } +/// Transaction ordering when requesting pending set. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum PendingOrdering { + /// Get pending transactions ordered by their priority (potentially expensive) + Priority, + /// Get pending transactions without any care of particular ordering (cheaper). + Unordered, +} + +/// Pending set query settings +#[derive(Debug, Clone)] +pub struct PendingSettings { + /// Current block number (affects readiness of some transactions). + pub block_number: u64, + /// Current timestamp (affects readiness of some transactions). + pub current_timestamp: u64, + /// Nonce cap (for dust protection; EIP-168) + pub nonce_cap: Option, + /// Maximal number of transactions in pending the set. + pub max_len: usize, + /// Ordering of transactions. + pub ordering: PendingOrdering, +} + +impl PendingSettings { + /// Get all transactions (no cap or len limit) prioritized. + pub fn all_prioritized(block_number: u64, current_timestamp: u64) -> Self { + PendingSettings { + block_number, + current_timestamp, + nonce_cap: None, + max_len: usize::max_value(), + ordering: PendingOrdering::Priority, + } + } +} + /// Transaction priority. #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub(crate) enum Priority { diff --git a/miner/src/pool/queue.rs b/miner/src/pool/queue.rs index cf4a956f426..4351087b29b 100644 --- a/miner/src/pool/queue.rs +++ b/miner/src/pool/queue.rs @@ -26,7 +26,10 @@ use parking_lot::RwLock; use transaction; use txpool::{self, Verifier}; -use pool::{self, scoring, verifier, client, ready, listener, PrioritizationStrategy}; +use pool::{ + self, scoring, verifier, client, ready, listener, + PrioritizationStrategy, PendingOrdering, PendingSettings, +}; use pool::local_transactions::LocalTransactionsList; type Listener = (LocalTransactionsList, (listener::Notifier, listener::Logger)); @@ -74,6 +77,7 @@ struct CachedPending { nonce_cap: Option, has_local_pending: bool, pending: Option>>, + max_len: usize, } impl CachedPending { @@ -85,6 +89,7 @@ impl CachedPending { has_local_pending: false, pending: None, nonce_cap: None, + max_len: 0, } } @@ -99,6 +104,7 @@ impl CachedPending { block_number: u64, current_timestamp: u64, nonce_cap: Option<&U256>, + max_len: usize, ) -> Option>> { // First check if we have anything in cache. let pending = self.pending.as_ref()?; @@ -123,7 +129,12 @@ impl CachedPending { return None; } - Some(pending.clone()) + // It's fine to just take a smaller subset, but not other way around. + if max_len > self.max_len { + return None; + } + + Some(pending.iter().take(max_len).cloned().collect()) } } @@ -173,7 +184,7 @@ impl TransactionQueue { transactions: Vec, ) -> Vec> { // Run verification - let _timer = ::trace_time::PerfTimer::new("pool::verify_and_import"); + trace_time!("pool::verify_and_import"); let options = self.options.read().clone(); let verifier = verifier::Verifier::new(client, options, self.insertion_id.clone()); @@ -203,13 +214,13 @@ impl TransactionQueue { results } - /// Returns all transactions in the queue ordered by priority. + /// Returns all transactions in the queue without explicit ordering. pub fn all_transactions(&self) -> Vec> { let ready = |_tx: &pool::VerifiedTransaction| txpool::Readiness::Ready; - self.pool.read().pending(ready).collect() + self.pool.read().unordered_pending(ready).collect() } - /// Returns current pneding transactions. + /// Returns current pending transactions ordered by priority. /// /// NOTE: This may return a cached version of pending transaction set. /// Re-computing the pending set is possible with `#collect_pending` method, @@ -217,24 +228,31 @@ impl TransactionQueue { pub fn pending( &self, client: C, - block_number: u64, - current_timestamp: u64, - nonce_cap: Option, + settings: PendingSettings, ) -> Vec> where C: client::NonceClient, { - - if let Some(pending) = self.cached_pending.read().pending(block_number, current_timestamp, nonce_cap.as_ref()) { + let PendingSettings { block_number, current_timestamp, nonce_cap, max_len, ordering } = settings; + if let Some(pending) = self.cached_pending.read().pending(block_number, current_timestamp, nonce_cap.as_ref(), max_len) { return pending; } // Double check after acquiring write lock let mut cached_pending = self.cached_pending.write(); - if let Some(pending) = cached_pending.pending(block_number, current_timestamp, nonce_cap.as_ref()) { + if let Some(pending) = cached_pending.pending(block_number, current_timestamp, nonce_cap.as_ref(), max_len) { return pending; } - let pending: Vec<_> = self.collect_pending(client, block_number, current_timestamp, nonce_cap, |i| i.collect()); + // In case we don't have a cached set, but we don't care about order + // just return the unordered set. + if let PendingOrdering::Unordered = ordering { + let ready = Self::ready(client, block_number, current_timestamp, nonce_cap); + return self.pool.read().unordered_pending(ready).take(max_len).collect(); + } + + let pending: Vec<_> = self.collect_pending(client, block_number, current_timestamp, nonce_cap, |i| { + i.take(max_len).collect() + }); *cached_pending = CachedPending { block_number, @@ -242,6 +260,7 @@ impl TransactionQueue { nonce_cap, has_local_pending: self.has_local_pending_transactions(), pending: Some(pending.clone()), + max_len, }; pending @@ -266,15 +285,27 @@ impl TransactionQueue { scoring::NonceAndGasPrice, Listener, >) -> T, + { + debug!(target: "txqueue", "Re-computing pending set for block: {}", block_number); + trace_time!("pool::collect_pending"); + let ready = Self::ready(client, block_number, current_timestamp, nonce_cap); + collect(self.pool.read().pending(ready)) + } + + fn ready( + client: C, + block_number: u64, + current_timestamp: u64, + nonce_cap: Option, + ) -> (ready::Condition, ready::State) where + C: client::NonceClient, { let pending_readiness = ready::Condition::new(block_number, current_timestamp); // don't mark any transactions as stale at this point. let stale_id = None; let state_readiness = ready::State::new(client, stale_id, nonce_cap); - let ready = (pending_readiness, state_readiness); - - collect(self.pool.read().pending(ready)) + (pending_readiness, state_readiness) } /// Culls all stalled transactions from the pool. @@ -415,6 +446,12 @@ impl TransactionQueue { let mut pool = self.pool.write(); (pool.listener_mut().1).0.add(f); } + + /// Check if pending set is cached. + #[cfg(test)] + pub fn is_pending_cached(&self) -> bool { + self.cached_pending.read().pending.is_some() + } } fn convert_error(err: txpool::Error) -> transaction::Error { @@ -440,7 +477,7 @@ mod tests { fn should_get_pending_transactions() { let queue = TransactionQueue::new(txpool::Options::default(), verifier::Options::default(), PrioritizationStrategy::GasPriceOnly); - let pending: Vec<_> = queue.pending(TestClient::default(), 0, 0, None); + let pending: Vec<_> = queue.pending(TestClient::default(), PendingSettings::all_prioritized(0, 0)); for tx in pending { assert!(tx.signed().nonce > 0.into()); diff --git a/miner/src/pool/tests/mod.rs b/miner/src/pool/tests/mod.rs index ac2e6b008e9..ef83db4a901 100644 --- a/miner/src/pool/tests/mod.rs +++ b/miner/src/pool/tests/mod.rs @@ -18,7 +18,7 @@ use ethereum_types::U256; use transaction::{self, PendingTransaction}; use txpool; -use pool::{verifier, TransactionQueue, PrioritizationStrategy}; +use pool::{verifier, TransactionQueue, PrioritizationStrategy, PendingSettings, PendingOrdering}; pub mod tx; pub mod client; @@ -108,7 +108,7 @@ fn should_handle_same_transaction_imported_twice_with_different_state_nonces() { // and then there should be only one transaction in current (the one with higher gas_price) assert_eq!(res, vec![Ok(())]); assert_eq!(txq.status().status.transaction_count, 1); - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top[0].hash, hash); } @@ -133,7 +133,7 @@ fn should_move_all_transactions_from_future() { // then assert_eq!(res, vec![Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top[0].hash, hash); assert_eq!(top[1].hash, hash2); } @@ -207,7 +207,7 @@ fn should_import_txs_from_same_sender() { txq.import(TestClient::new(), txs.local().into_vec()); // then - let top = txq.pending(TestClient::new(), 0 ,0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0 ,0)); assert_eq!(top[0].hash, hash); assert_eq!(top[1].hash, hash2); assert_eq!(top.len(), 2); @@ -229,7 +229,7 @@ fn should_prioritize_local_transactions_within_same_nonce_height() { assert_eq!(res, vec![Ok(()), Ok(())]); // then - let top = txq.pending(client, 0, 0, None); + let top = txq.pending(client, PendingSettings::all_prioritized(0, 0)); assert_eq!(top[0].hash, hash); // local should be first assert_eq!(top[1].hash, hash2); assert_eq!(top.len(), 2); @@ -251,7 +251,7 @@ fn should_prioritize_reimported_transactions_within_same_nonce_height() { assert_eq!(res, vec![Ok(()), Ok(())]); // then - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top[0].hash, hash); // retracted should be first assert_eq!(top[1].hash, hash2); assert_eq!(top.len(), 2); @@ -270,7 +270,7 @@ fn should_not_prioritize_local_transactions_with_different_nonce_height() { assert_eq!(res, vec![Ok(()), Ok(())]); // then - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top[0].hash, hash); assert_eq!(top[1].hash, hash2); assert_eq!(top.len(), 2); @@ -288,7 +288,7 @@ fn should_put_transaction_to_futures_if_gap_detected() { // then assert_eq!(res, vec![Ok(()), Ok(())]); - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top.len(), 1); assert_eq!(top[0].hash, hash); } @@ -308,9 +308,9 @@ fn should_handle_min_block() { assert_eq!(res, vec![Ok(()), Ok(())]); // then - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top.len(), 0); - let top = txq.pending(TestClient::new(), 1, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(1, 0)); assert_eq!(top.len(), 2); } @@ -341,7 +341,7 @@ fn should_move_transactions_if_gap_filled() { let res = txq.import(TestClient::new(), vec![tx, tx2].local()); assert_eq!(res, vec![Ok(()), Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); // when let res = txq.import(TestClient::new(), vec![tx1.local()]); @@ -349,7 +349,7 @@ fn should_move_transactions_if_gap_filled() { // then assert_eq!(txq.status().status.transaction_count, 3); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 3); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 3); } #[test] @@ -361,12 +361,12 @@ fn should_remove_transaction() { let res = txq.import(TestClient::default(), vec![tx, tx2].local()); assert_eq!(res, vec![Ok(()), Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); // when txq.cull(TestClient::new().with_nonce(124)); assert_eq!(txq.status().status.transaction_count, 1); - assert_eq!(txq.pending(TestClient::new().with_nonce(125), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new().with_nonce(125), PendingSettings::all_prioritized(0, 0)).len(), 1); txq.cull(TestClient::new().with_nonce(126)); // then @@ -384,19 +384,19 @@ fn should_move_transactions_to_future_if_gap_introduced() { let res = txq.import(TestClient::new(), vec![tx3, tx2].local()); assert_eq!(res, vec![Ok(()), Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); let res = txq.import(TestClient::new(), vec![tx].local()); assert_eq!(res, vec![Ok(())]); assert_eq!(txq.status().status.transaction_count, 3); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 3); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 3); // when txq.remove(vec![&hash], true); // then assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); } #[test] @@ -447,7 +447,7 @@ fn should_prefer_current_transactions_when_hitting_the_limit() { assert_eq!(res, vec![Ok(())]); assert_eq!(txq.status().status.transaction_count, 1); - let top = txq.pending(TestClient::new(), 0, 0, None); + let top = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); assert_eq!(top.len(), 1); assert_eq!(top[0].hash, hash); assert_eq!(txq.next_nonce(TestClient::new(), &sender), Some(124.into())); @@ -494,19 +494,19 @@ fn should_accept_same_transaction_twice_if_removed() { let res = txq.import(TestClient::new(), txs.local().into_vec()); assert_eq!(res, vec![Ok(()), Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 2); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 2); // when txq.remove(vec![&hash], true); assert_eq!(txq.status().status.transaction_count, 1); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 0); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 0); let res = txq.import(TestClient::new(), vec![tx1].local()); assert_eq!(res, vec![Ok(())]); // then assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 2); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 2); } #[test] @@ -526,8 +526,8 @@ fn should_not_replace_same_transaction_if_the_fee_is_less_than_minimal_bump() { // then assert_eq!(res, vec![Err(transaction::Error::TooCheapToReplace), Ok(())]); assert_eq!(txq.status().status.transaction_count, 2); - assert_eq!(txq.pending(client.clone(), 0, 0, None)[0].signed().gas_price, U256::from(20)); - assert_eq!(txq.pending(client.clone(), 0, 0, None)[1].signed().gas_price, U256::from(2)); + assert_eq!(txq.pending(client.clone(), PendingSettings::all_prioritized(0, 0))[0].signed().gas_price, U256::from(20)); + assert_eq!(txq.pending(client.clone(), PendingSettings::all_prioritized(0, 0))[1].signed().gas_price, U256::from(2)); } #[test] @@ -569,7 +569,7 @@ fn should_return_valid_last_nonce_after_cull() { let client = TestClient::new().with_nonce(124); txq.cull(client.clone()); // tx2 should be not be promoted to current - assert_eq!(txq.pending(client.clone(), 0, 0, None).len(), 0); + assert_eq!(txq.pending(client.clone(), PendingSettings::all_prioritized(0, 0)).len(), 0); // then assert_eq!(txq.next_nonce(client.clone(), &sender), None); @@ -667,7 +667,7 @@ fn should_accept_local_transactions_below_min_gas_price() { assert_eq!(res, vec![Ok(())]); // then - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); } #[test] @@ -685,7 +685,7 @@ fn should_accept_local_service_transaction() { assert_eq!(res, vec![Ok(())]); // then - assert_eq!(txq.pending(TestClient::new(), 0, 0, None).len(), 1); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)).len(), 1); } #[test] @@ -726,15 +726,77 @@ fn should_not_return_transactions_over_nonce_cap() { assert_eq!(res, vec![Ok(()), Ok(()), Ok(())]); // when - let all = txq.pending(TestClient::new(), 0, 0, None); + let all = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); // This should invalidate the cache! - let limited = txq.pending(TestClient::new(), 0, 0, Some(123.into())); + let limited = txq.pending(TestClient::new(), PendingSettings { + block_number: 0, + current_timestamp: 0, + nonce_cap: Some(123.into()), + max_len: usize::max_value(), + ordering: PendingOrdering::Priority, + }); // then assert_eq!(all.len(), 3); assert_eq!(limited.len(), 1); } +#[test] +fn should_return_cached_pending_even_if_unordered_is_requested() { + // given + let txq = new_queue(); + let tx1 = Tx::default().signed(); + let (tx2_1, tx2_2)= Tx::default().signed_pair(); + let tx2_1_hash = tx2_1.hash(); + let res = txq.import(TestClient::new(), vec![tx1].unverified()); + assert_eq!(res, vec![Ok(())]); + let res = txq.import(TestClient::new(), vec![tx2_1, tx2_2].local()); + assert_eq!(res, vec![Ok(()), Ok(())]); + + // when + let all = txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 0)); + assert_eq!(all[0].hash, tx2_1_hash); + assert_eq!(all.len(), 3); + + // This should not invalidate the cache! + let limited = txq.pending(TestClient::new(), PendingSettings { + block_number: 0, + current_timestamp: 0, + nonce_cap: None, + max_len: 3, + ordering: PendingOrdering::Unordered, + }); + + // then + assert_eq!(all, limited); +} + +#[test] +fn should_return_unordered_and_not_populate_the_cache() { + // given + let txq = new_queue(); + let tx1 = Tx::default().signed(); + let (tx2_1, tx2_2)= Tx::default().signed_pair(); + let res = txq.import(TestClient::new(), vec![tx1].unverified()); + assert_eq!(res, vec![Ok(())]); + let res = txq.import(TestClient::new(), vec![tx2_1, tx2_2].local()); + assert_eq!(res, vec![Ok(()), Ok(())]); + + // when + // This should not invalidate the cache! + let limited = txq.pending(TestClient::new(), PendingSettings { + block_number: 0, + current_timestamp: 0, + nonce_cap: None, + max_len: usize::max_value(), + ordering: PendingOrdering::Unordered, + }); + + // then + assert_eq!(limited.len(), 3); + assert!(!txq.is_pending_cached()); +} + #[test] fn should_clear_cache_after_timeout_for_local() { // given @@ -748,12 +810,12 @@ fn should_clear_cache_after_timeout_for_local() { // This should populate cache and set timestamp to 1 // when - assert_eq!(txq.pending(TestClient::new(), 0, 1, None).len(), 0); - assert_eq!(txq.pending(TestClient::new(), 0, 1000, None).len(), 0); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 1)).len(), 0); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 1000)).len(), 0); // This should invalidate the cache and trigger transaction ready. // then - assert_eq!(txq.pending(TestClient::new(), 0, 1002, None).len(), 2); + assert_eq!(txq.pending(TestClient::new(), PendingSettings::all_prioritized(0, 1002)).len(), 2); } #[test] diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index ce30f3cd85e..31af69eaa06 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -304,7 +304,7 @@ impl FullDependencies { let client = EthPubSubClient::new(self.client.clone(), self.remote.clone()); let h = client.handler(); self.miner.add_transactions_listener(Box::new(move |hashes| if let Some(h) = h.upgrade() { - h.new_transactions(hashes); + h.notify_new_transactions(hashes); })); if let Some(h) = client.handler().upgrade() { @@ -525,7 +525,7 @@ impl LightDependencies { let h = client.handler(); self.transaction_queue.write().add_listener(Box::new(move |transactions| { if let Some(h) = h.upgrade() { - h.new_transactions(transactions); + h.notify_new_transactions(transactions); } })); handler.extend_with(EthPubSub::to_delegate(client)); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 338a8926829..d227f45f535 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -36,7 +36,7 @@ jsonrpc-macros = { git = "https://github.com/paritytech/jsonrpc.git", branch = " jsonrpc-pubsub = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" } ethash = { path = "../ethash" } -ethcore = { path = "../ethcore" } +ethcore = { path = "../ethcore", features = ["test-helpers"] } ethcore-bytes = { path = "../util/bytes" } ethcore-crypto = { path = "../ethcore/crypto" } ethcore-devtools = { path = "../devtools" } diff --git a/rpc/src/v1/impls/eth_filter.rs b/rpc/src/v1/impls/eth_filter.rs index bbad2fe27d0..c91070cbfb6 100644 --- a/rpc/src/v1/impls/eth_filter.rs +++ b/rpc/src/v1/impls/eth_filter.rs @@ -85,7 +85,7 @@ impl Filterable for EthFilterClient where } fn pending_transactions_hashes(&self) -> Vec { - self.miner.ready_transactions(&*self.client) + self.miner.ready_transactions(&*self.client, usize::max_value(), miner::PendingOrdering::Priority) .into_iter() .map(|tx| tx.signed().hash()) .collect() diff --git a/rpc/src/v1/impls/eth_pubsub.rs b/rpc/src/v1/impls/eth_pubsub.rs index 11fef2e0bd0..38162e8ea1b 100644 --- a/rpc/src/v1/impls/eth_pubsub.rs +++ b/rpc/src/v1/impls/eth_pubsub.rs @@ -175,7 +175,7 @@ impl ChainNotificationHandler { } /// Notify all subscribers about new transaction hashes. - pub fn new_transactions(&self, hashes: &[H256]) { + pub fn notify_new_transactions(&self, hashes: &[H256]) { for subscriber in self.transactions_subscribers.read().values() { for hash in hashes { Self::notify(&self.remote, subscriber, pubsub::Result::TransactionHash((*hash).into())); diff --git a/rpc/src/v1/impls/light/parity.rs b/rpc/src/v1/impls/light/parity.rs index 91db00ca309..6e93132b922 100644 --- a/rpc/src/v1/impls/light/parity.rs +++ b/rpc/src/v1/impls/light/parity.rs @@ -264,12 +264,13 @@ impl Parity for ParityClient { .map(Into::into) } - fn pending_transactions(&self) -> Result> { + fn pending_transactions(&self, limit: Trailing) -> Result> { let txq = self.light_dispatch.transaction_queue.read(); let chain_info = self.light_dispatch.client.chain_info(); Ok( txq.ready_transactions(chain_info.best_block_number, chain_info.best_block_timestamp) .into_iter() + .take(limit.unwrap_or_else(usize::max_value)) .map(|tx| Transaction::from_pending(tx, chain_info.best_block_number, self.eip86_transition)) .collect::>() ) diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index f18723eafe1..d7c26014edd 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -303,15 +303,19 @@ impl Parity for ParityClient where .map(Into::into) } - fn pending_transactions(&self) -> Result> { + fn pending_transactions(&self, limit: Trailing) -> Result> { let block_number = self.client.chain_info().best_block_number; - let ready_transactions = self.miner.ready_transactions(&*self.client); + let ready_transactions = self.miner.ready_transactions( + &*self.client, + limit.unwrap_or_else(usize::max_value), + miner::PendingOrdering::Priority, + ); Ok(ready_transactions .into_iter() .map(|t| Transaction::from_pending(t.pending().clone(), block_number, self.eip86_transition)) .collect() - ) + ) } fn all_transactions(&self) -> Result> { diff --git a/rpc/src/v1/tests/helpers/miner_service.rs b/rpc/src/v1/tests/helpers/miner_service.rs index 90201e346a5..8d0ec23ae1c 100644 --- a/rpc/src/v1/tests/helpers/miner_service.rs +++ b/rpc/src/v1/tests/helpers/miner_service.rs @@ -27,7 +27,7 @@ use ethcore::engines::EthEngine; use ethcore::error::Error; use ethcore::header::{BlockNumber, Header}; use ethcore::ids::BlockId; -use ethcore::miner::{MinerService, AuthoringParams}; +use ethcore::miner::{self, MinerService, AuthoringParams}; use ethcore::receipt::{Receipt, RichReceipt}; use ethereum_types::{H256, U256, Address}; use miner::pool::local_transactions::Status as LocalTransactionStatus; @@ -208,7 +208,7 @@ impl MinerService for TestMinerService { self.local_transactions.lock().iter().map(|(hash, stats)| (*hash, stats.clone())).collect() } - fn ready_transactions(&self, _chain: &C) -> Vec> { + fn ready_transactions(&self, _chain: &C, _max_len: usize, _ordering: miner::PendingOrdering) -> Vec> { self.queued_transactions() } diff --git a/rpc/src/v1/tests/mocked/eth_pubsub.rs b/rpc/src/v1/tests/mocked/eth_pubsub.rs index 0d886fe2f1f..30c99fc67ac 100644 --- a/rpc/src/v1/tests/mocked/eth_pubsub.rs +++ b/rpc/src/v1/tests/mocked/eth_pubsub.rs @@ -181,7 +181,7 @@ fn should_subscribe_to_pending_transactions() { assert_eq!(io.handle_request_sync(request, metadata.clone()), Some(response.to_owned())); // Send new transactions - handler.new_transactions(&[5.into(), 7.into()]); + handler.notify_new_transactions(&[5.into(), 7.into()]); let (res, receiver) = receiver.into_future().wait().unwrap(); let response = r#"{"jsonrpc":"2.0","method":"eth_subscription","params":{"result":"0x0000000000000000000000000000000000000000000000000000000000000005","subscription":"0x416d77337e24399d"}}"#; diff --git a/rpc/src/v1/traits/parity.rs b/rpc/src/v1/traits/parity.rs index f78cf8052eb..1b9a7d09f5b 100644 --- a/rpc/src/v1/traits/parity.rs +++ b/rpc/src/v1/traits/parity.rs @@ -141,7 +141,7 @@ build_rpc_trait! { /// Returns all pending transactions from transaction queue. #[rpc(name = "parity_pendingTransactions")] - fn pending_transactions(&self) -> Result>; + fn pending_transactions(&self, Trailing) -> Result>; /// Returns all transactions from transaction queue. /// diff --git a/transaction-pool/Cargo.toml b/transaction-pool/Cargo.toml index 8965c8cee01..0ba1790a47f 100644 --- a/transaction-pool/Cargo.toml +++ b/transaction-pool/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Generic transaction pool." name = "transaction-pool" -version = "1.12.0" +version = "1.12.1" license = "GPL-3.0" authors = ["Parity Technologies "] diff --git a/transaction-pool/src/pool.rs b/transaction-pool/src/pool.rs index dcd52a3e7e6..4bbf00ef240 100644 --- a/transaction-pool/src/pool.rs +++ b/transaction-pool/src/pool.rs @@ -15,7 +15,8 @@ // along with Parity. If not, see . use std::sync::Arc; -use std::collections::{HashMap, BTreeSet}; +use std::slice; +use std::collections::{hash_map, HashMap, BTreeSet}; use error; use listener::{Listener, NoopListener}; @@ -416,7 +417,16 @@ impl Pool where PendingIterator { ready, best_transactions, - pool: self + pool: self, + } + } + + /// Returns unprioritized list of ready transactions. + pub fn unordered_pending>(&self, ready: R) -> UnorderedIterator { + UnorderedIterator { + ready, + senders: self.transactions.iter(), + transactions: None, } } @@ -482,6 +492,50 @@ impl Pool where } } +/// An iterator over all pending (ready) transactions in unoredered fashion. +/// +/// NOTE: Current implementation will iterate over all transactions from particular sender +/// ordered by nonce, but that might change in the future. +/// +/// NOTE: the transactions are not removed from the queue. +/// You might remove them later by calling `cull`. +pub struct UnorderedIterator<'a, T, R, S> where + T: VerifiedTransaction + 'a, + S: Scoring + 'a, +{ + ready: R, + senders: hash_map::Iter<'a, T::Sender, Transactions>, + transactions: Option>>, +} + +impl<'a, T, R, S> Iterator for UnorderedIterator<'a, T, R, S> where + T: VerifiedTransaction, + R: Ready, + S: Scoring, +{ + type Item = Arc; + + fn next(&mut self) -> Option { + loop { + if let Some(transactions) = self.transactions.as_mut() { + if let Some(tx) = transactions.next() { + match self.ready.is_ready(&tx) { + Readiness::Ready => { + return Some(tx.transaction.clone()); + }, + state => trace!("[{:?}] Ignoring {:?} transaction.", tx.hash(), state), + } + } + } + + // otherwise fallback and try next sender + let next_sender = self.senders.next()?; + self.transactions = Some(next_sender.1.iter()); + } + } +} + + /// An iterator over all pending (ready) transactions. /// NOTE: the transactions are not removed from the queue. /// You might remove them later by calling `cull`. diff --git a/transaction-pool/src/tests/mod.rs b/transaction-pool/src/tests/mod.rs index 6edd60e60e2..77c25287570 100644 --- a/transaction-pool/src/tests/mod.rs +++ b/transaction-pool/src/tests/mod.rs @@ -250,6 +250,66 @@ fn should_construct_pending() { assert_eq!(pending.next(), None); } +#[test] +fn should_return_unordered_iterator() { + // given + let b = TransactionBuilder::default(); + let mut txq = TestPool::default(); + + let tx0 = txq.import(b.tx().nonce(0).gas_price(5).new()).unwrap(); + let tx1 = txq.import(b.tx().nonce(1).gas_price(5).new()).unwrap(); + let tx2 = txq.import(b.tx().nonce(2).new()).unwrap(); + let tx3 = txq.import(b.tx().nonce(3).gas_price(4).new()).unwrap(); + //gap + txq.import(b.tx().nonce(5).new()).unwrap(); + + let tx5 = txq.import(b.tx().sender(1).nonce(0).new()).unwrap(); + let tx6 = txq.import(b.tx().sender(1).nonce(1).new()).unwrap(); + let tx7 = txq.import(b.tx().sender(1).nonce(2).new()).unwrap(); + let tx8 = txq.import(b.tx().sender(1).nonce(3).gas_price(4).new()).unwrap(); + // gap + txq.import(b.tx().sender(1).nonce(5).new()).unwrap(); + + let tx9 = txq.import(b.tx().sender(2).nonce(0).new()).unwrap(); + assert_eq!(txq.light_status().transaction_count, 11); + assert_eq!(txq.status(NonceReady::default()), Status { + stalled: 0, + pending: 9, + future: 2, + }); + assert_eq!(txq.status(NonceReady::new(1)), Status { + stalled: 3, + pending: 6, + future: 2, + }); + + // when + let all: Vec<_> = txq.unordered_pending(NonceReady::default()).collect(); + + let chain1 = vec![tx0, tx1, tx2, tx3]; + let chain2 = vec![tx5, tx6, tx7, tx8]; + let chain3 = vec![tx9]; + + assert_eq!(all.len(), chain1.len() + chain2.len() + chain3.len()); + + let mut options = vec![ + vec![chain1.clone(), chain2.clone(), chain3.clone()], + vec![chain2.clone(), chain1.clone(), chain3.clone()], + vec![chain2.clone(), chain3.clone(), chain1.clone()], + vec![chain3.clone(), chain2.clone(), chain1.clone()], + vec![chain3.clone(), chain1.clone(), chain2.clone()], + vec![chain1.clone(), chain3.clone(), chain2.clone()], + ].into_iter().map(|mut v| { + let mut first = v.pop().unwrap(); + for mut x in v { + first.append(&mut x); + } + first + }); + + assert!(options.any(|opt| all == opt)); +} + #[test] fn should_update_scoring_correctly() { // given From 0bb78814a60d1abc759ded380906a79cf3474f78 Mon Sep 17 00:00:00 2001 From: Vladyslav Lupashevskyi Date: Tue, 12 Jun 2018 10:31:14 +0300 Subject: [PATCH 5/5] Tx permission contract improvement (#8400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Tx permission contract improvement * Use tuple for to address * Introduced ABI for deprecated tx permission contract * Improved ABI for tx permission contract with contract name, name hash and version * Introduced support for deprecated tx permission contract + fixed cache for the new version + introduced `target` for tx filter loging * Introduced test for the new tx permission contract version + old test renamed as deprecated * Removed empty lines * Introduced filter_only_sender return value in allowedTxTypes fn + improved caching * Introduced version checking for tx permission contract * Moved tx permission contract test genesis specs to separate files * handle queue import errors a bit more gracefully (#8385) * Some tweaks to main.rs for parity as a library (#8370) * Some tweaks to main.rs for parity as a library * Remove pub from PostExecutionAction * New Transaction Queue implementation (#8074) * Implementation of Verifier, Scoring and Ready. * Queue in progress. * TransactionPool. * Prepare for txpool release. * Miner refactor [WiP] * WiP reworking miner. * Make it compile. * Add some docs. * Split blockchain access to a separate file. * Work on miner API. * Fix ethcore tests. * Refactor miner interface for sealing/work packages. * Implement next nonce. * RPC compiles. * Implement couple of missing methdods for RPC. * Add transaction queue listeners. * Compiles! * Clean-up and parallelize. * Get rid of RefCell in header. * Revert "Get rid of RefCell in header." This reverts commit 0f2424c9b7319a786e1565ea2a8a6d801a21b4fb. * Override Sync requirement. * Fix status display. * Unify logging. * Extract some cheap checks. * Measurements and optimizations. * Fix scoring bug, heap size of bug and add cache * Disable tx queueing and parallel verification. * Make ethcore and ethcore-miner compile again. * Make RPC compile again. * Bunch of txpool tests. * Migrate transaction queue tests. * Nonce Cap * Nonce cap cache and tests. * Remove stale future transactions from the queue. * Optimize scoring and write some tests. * Simple penalization. * Clean up and support for different scoring algorithms. * Add CLI parameters for the new queue. * Remove banning queue. * Disable debug build. * Change per_sender limit to be 1% instead of 5% * Avoid cloning when propagating transactions. * Remove old todo. * Post-review fixes. * Fix miner options default. * Implement back ready transactions for light client. * Get rid of from_pending_block * Pass rejection reason. * Add more details to drop. * Rollback heap size of. * Avoid cloning hashes when propagating and include more details on rejection. * Fix tests. * Introduce nonces cache. * Remove uneccessary hashes allocation. * Lower the mem limit. * Re-enable parallel verification. * Add miner log. Don't check the type if not below min_gas_price. * Add more traces, fix disabling miner. * Fix creating pending blocks twice on AuRa authorities. * Fix tests. * re-use pending blocks in AuRa * Use reseal_min_period to prevent too frequent update_sealing. * Fix log to contain hash not sender. * Optimize local transactions. * Fix aura tests. * Update locks comments. * Get rid of unsafe Sync impl. * Review fixes. * Remove excessive matches. * Fix compilation errors. * Use new pool in private transactions. * Fix private-tx test. * Fix secret store tests. * Actually use gas_floor_target * Fix config tests. * Fix pool tests. * Address grumbles. * clarify that windows need perl and yasm (#8402) * Unify and limit rocksdb dependency places (#8371) * secret_store: remove kvdb_rocksdb dependency * cli: init db mod for open dispatch * cli: move db, client_db, restoration_db, secretstore_db to a separate mod * migration: rename to migration-rocksdb and remove ethcore-migrations * ethcore: re-move kvdb-rocksdb dep to test * mark test_helpers as test only and fix migration mod naming * Move restoration_db_handler to test_helpers_internal * Fix missing preambles in test_helpers_internal and rocksdb/helpers * Move test crates downward * Fix missing docs * cli, db::open_db: move each argument to a separate line * Use featuregate instead of dead code for `open_secretstore_db` * Move pathbuf import to open_secretstore_db Because it's only used there behind a feature gate * Use tokio::spawn in secret_store listener and fix Uri (#8373) * Directly wait for future to resolve in a threadpool * Ignore return value * Use path.starts_with instead of req_uri.is_absolute The later now means something else in hyper 0.11.. * Use tokio::spawn * typo: remove accidential unsafe impl * remove Tendermint extra_info due to seal inconsistencies (#8367) * More code refactoring to integrate Duration (#8322) * More code refactoring to integrate Duration * Fix typo * Fix tests * More test fix * tokio-core v0.1.16 -> v0.1.17 (#8408) * Replace legacy Rlp with UntrustedRlp and use in ethcore rlp views (#8316) * WIP * Replace Rlp with UntrustedRlp in views, explicity unwrap with expect First pass to get it to compile. Need to figure out whether to do this or to propogate Errors upstream, which would require many more changes to dependent code. If we do this way we are assuming that the views are always used in a context where the rlp is trusted to be valid e.g. when reading from our own DB. So need to fid out whether views are used with data received from an untrusted (e.g. extrernal peer). * Remove original Rlp impl, rename UntrustedRlp -> Rlp * Create rlp views with view! macro to record debug info Views are assumed to be over valid rlp, so if there is a decoding error we record where the view was created in the first place and report it in the expect * Use $crate in view! macro to avoid import, fix tests * Expect valid rlp in decode functions for now * Replace spaces with tabs in new file * Add doc tests for creating views with macro * Update rlp docs to reflect removing of UntrustedRlp * Replace UntrustedRlp usages in private-tx merge * Fix TODO comments (#8413) * update zip to 0.3 (#8381) * update zip to 0.3 * enable zip deflate feature * typo, docs parity_chainId: empty string -> None (#8434) * Fix receipts stripping. (#8414) * Changelogs for 1.9.6 and 1.10.1 (#8411) * Add changelog for 1.9.6 * Add Changelog for 1.10.1 * Move ethcore::Error to error_chain (#8386) * WIP * Convert Ethcore error to use error_chain * Use error_chain for ImportError and BlockImportError * Fix error pattern matches for error_chain in miner * Implement explicit From for AccountsError * Fix pattern matches for ErrorKinds * Handle ethcore error_chain in light client * Explicitly define Result type to avoid shadowing * Fix remaining Error pattern matches * Fix tab space formatting * Helps if the tests compile * Fix error chain matching after merge * remove From::from. (#8390) * Some tiny modifications. 1. fix some typo in the comment. 2. sort the order of methods in 'impl state::Backend for StateDB` * Remove the clone of code_cache, as it has been done in clone_basic. * remove From::from. It seems not necessary. * Use forked app_dirs crate for reverted Windows dir behavior (#8438) * Remove unused appdirs dependency in CLI * Use forked app_dirs crate for reverted Windows dir behavior * Permission fix (#8441) * Block reward contract (#8419) * engine: add block reward contract abi and helper client * aura: add support for block reward contract * engine: test block reward contract client * aura: test block reward contract * engine + aura: add missing docs * engine: share SystemCall type alias * aura: add transition for block reward contract * engine: fix example block reward contract source link and bytecode * Improve VM executor stack size estimation rules (#8439) * Improve VM executor stack size estimation rules * typo: docs add "(Debug build)" comment * Fix an off by one typo and set minimal stack size This avoids the case if `depth_threshold == max_depth`. Usually setting stack size to zero will just rebound it to platform minimal stack size, but we set it here just in case. * Use saturating_sub to avoid potential overflow * Private transactions processing error handling (#8431) * Integration test for private transaction returned * Do not interrupt verification in case of errors * Helpers use specified * Review comments fixed * Update Cargo hidapi-rs dependency (#8447) * Allow 32 bit pipelines to fail (#8454) * Disable 32bit tragets for gitlab * Rename linux pipelines * Update wasmi (#8452) * Return error in case eth_call returns VM errors (#8448) * Add VMError generator * Return executed exceptions in eth_call * ParityShell::open `Return result` (#8377) * start * add error handling for winapi * fix typo * fix warnings and windows errors * formatting * Address review comments * fix docker build (#8462) * Add changelog for 1.9.7 and 1.10.2 (#8460) * Add changelog for 1.9.7 * Add Changelog for 1.10.2 * Apply proper markdown * Run a spellchecker :) * Be pedantic about the 32-bit pipelines :) * fix typos in vm description comment (#8446) * Use rename_all for RichBlock and RichHeader serialization (#8471) * typo: fix a resolved TODO comment * Use rename_all instead of individual renames * Don't require write lock when fetching status. (#8481) * Bump master to 1.12 (#8477) * Bump master to 1.12 * Bump crates to 1.12 * Bump mac installer version to 1.12 * Update Gitlab scripts * Fix snap builds (#8483) * Update hardcodedSync for Ethereum, Kovan, and Ropsten (#8489) * Update wasmi and pwasm-utils (#8493) * Update wasmi to 0.2 New wasmi supports 32bit platforms and no longer requires a special feature to build for such platforms. * Update pwasm-utils to 0.1.5 * Remove three old warp boot nodes. (#8497) * Return error if RLP size of transaction exceeds the limit (#8473) * Return error if RLP size of transaction exceeds the limit * Review comments fixed * RLP check moved to verifier, corresponding pool test added * `duration_ns: u64 -> duration: Duration` (#8457) * duration_ns: u64 -> duration: Duration * format on millis {:.2} -> {} * Remove unused dependency `bigint` (#8505) * remove unused dependency bigint in * remove bigint in rpc_cli * Show imported messages for light client (#8517) * Directly return None if tracing is disabled (#8504) * Directly return None if tracing is disabled * Address gumbles: release read locks as fast as possible * Hardware Wallet trait (#8071) * getting started with replacing HardwareWalletManager * trezor and ledger impls the new trait with some drawbacks * Everything move to the new trait * It required lifetime annotations in the trait because [u8] in unsized * Lets now start moving entry point from HardwareWalletManager * rename trait to Wallet * move thread management to the actual wallets * Moved thread management to each respective Wallet * Cleaned up pub items that is needed to be pub * Wallet trait more or less finished * Cleaned up docs * fix tests * omit removed docs * fix spelling, naming och remove old comments * ledger test is broken, add correct logging format * So locally on my machine Linux Ubuntu 17.10 the test doesn't panic but on the CI server libusb::Context::new() fails which I don't understand because it has worked before * Additionally the ledger test is optional so I lean toward ignoring it the CI Server * ignore hardware tests by default * more verbose checking in ledger test * SecretStore: merge two types of errors into single one + Error::is_non_fatal (#8357) * SecretStore: error unify initial commit SecretStore: pass real error in error messages SecretStore: is_internal_error -> Error::is_non_fatal warnings SecretStore: ConsensusTemporaryUnreachable fix after merge removed comments removed comments SecretStore: updated HTTP error responses SecretStore: more ConsensusTemporaryUnreachable tests fix after rebase * fixed grumbles * use HashSet in tests * Enable WebAssembly and Byzantium for Ellaism (#8520) * Enable WebAssembly and Byzantium for Ellaism * Fix indentation * Remove empty lines * More changes for Android (#8421) * Transaction Pool improvements (#8470) * Don't use ethereum_types in transaction pool. * Hide internal insertion_id. * Fix tests. * Review grumbles. * Fetching logs by hash in blockchain database (#8463) * Fetch logs by hash in blockchain database * Fix tests * Add unit test for branch block logs fetching * Add docs that blocks must already be sorted * Handle branch block cases properly * typo: empty -> is_empty * Remove return_empty_if_none by using a closure * Use BTreeSet to avoid sorting again * Move is_canon to BlockChain * typo: pass value by reference * Use loop and wrap inside blocks to simplify the code Borrowed from https://github.com/paritytech/parity/pull/8463#discussion_r183453326 * typo: missed a comment * Pass on storage keys tracing to handle the case when it is not modified (#8491) * Pass on storage keys even if it is not modified * typo: account and storage query `to_pod_diff` builds both `touched_addresses` merge and storage keys merge. * Fix tests * Use state query directly because of suicided accounts * Fix a RefCell borrow issue * Add tests for unmodified storage trace * Address grumbles * typo: remove unwanted empty line * ensure_cached compiles with the original signature * Don't panic in import_block if invalid rlp (#8522) * Don't panic in import_block if invalid rlp * Remove redundant type annotation * Replace RLP header view usage with safe decoding Using the view will panic with invalid RLP. Here we use Rlp decoding directly which will return a `Result<_, DecoderError>`. While this path currently should not have any invalid RLP - it makes it safer if ever called with invalid RLP from other code paths. * Remove expect (#8536) * Remove expect and propagate rlp::DecoderErrors as TrieErrors * EIP 145: Bitwise shifting instructions in EVM (#8451) * Add SHL, SHR, SAR opcodes * Add have_bitwise_shifting schedule flag * Add all EIP tests for SHL * Add SHR implementation and tests * Implement SAR and add tests * Add eip145transition config param * Change map_or to map_or_else when possible * Consolidate crypto functionality in `ethcore-crypto`. (#8432) * Consolidate crypto functionality in `ethcore-crypto`. - Move `ecdh`/`ecies` modules to `ethkey`. - Refactor `ethcore-crypto` to use file per module. - Replace `subtle` with `ethcore_crypto::is_equal`. - Add `aes_gcm` module to `ethcore-crypto`. * Rename `aes::{encrypt,decrypt,decrypt_cbc}` ... ... to `aes::{encrypt_128_ctr,decrypt_128_ctr,decrypt_128_cbc}`. * ethcore, rpc, machine: refactor block reward application and tracing (#8490) * Keep all enacted blocks notify in order (#8524) * Keep all enacted blocks notify in order * Collect is unnecessary * Update ChainNotify to use ChainRouteType * Fix all ethcore fn defs * Wrap the type within ChainRoute * Fix private-tx and sync api * Fix secret_store API * Fix updater API * Fix rpc api * Fix informant api * Eagerly cache enacted/retracted and remove contain_enacted/retracted * Fix indent * tests: should use full expr form for struct constructor * Use into_enacted_retracted to further avoid copy * typo: not a function * rpc/tests: ChainRoute -> ChainRoute::new * Node table sorting according to last contact data (#8541) * network-devp2p: sort nodes in node table using last contact data * network-devp2p: rename node contact types in node table json output * network-devp2p: fix node table tests * network-devp2p: note node failure when failed to establish connection * network-devp2p: handle UselessPeer error * network-devp2p: note failure when marking node as useless * Rlp decode returns Result (#8527) rlp::decode returns Result Make a best effort to handle decoding errors gracefully throughout the code, using `expect` where the value is guaranteed to be valid (and in other places where it makes sense). * Parity as a library (#8412) * Parity as a library * Fix concerns * Allow using a null on_client_restart_cb * Fix more concerns * Test the C library in test.sh * Reduce CMake version to 3.5 * Move the clib test before cargo test * Add println in test * Trace precompiled contracts when the transfer value is not zero (#8486) * Trace precompiled contracts when the transfer value is not zero * Add tests for precompiled CALL tracing * Use byzantium test machine for the new test * Add notes in comments on why we don't trace all precompileds * Use is_transferred instead of transferred * Don't block sync when importing old blocks (#8530) * Alter IO queueing. * Don't require IoMessages to be Clone * Ancient blocks imported via IoChannel. * Get rid of private transactions io message. * Get rid of deadlock and fix disconnected handler. * Revert to old disconnect condition. * Fix tests. * Fix deadlock. * Make trace-time publishable. (#8568) * Remove State::replace_backend (#8569) * Refactoring `ethcore-sync` - Fixing warp-sync barrier (#8543) * Start dividing sync chain : first supplier method * WIP - updated chain sync supplier * Finish refactoring the Chain Sync Supplier * Create Chain Sync Requester * Add Propagator for Chain Sync * Add the Chain Sync Handler * Move tests from mod -> handler * Move tests to propagator * Refactor SyncRequester arguments * Refactoring peer fork header handler * Fix wrong highest block number in snapshot sync * Small refactor... * Address PR grumbles * Retry failed CI job * Fix tests * PR Grumbles * Decoding headers can fail (#8570) * rlp::decode returns Result * Fix journaldb to handle rlp::decode Result * Fix ethcore to work with rlp::decode returning Result * Light client handles rlp::decode returning Result * Fix tests in rlp_derive * Fix tests * Cleanup * cleanup * Allow panic rather than breaking out of iterator * Let decoding failures when reading from disk blow up * syntax * Fix the trivial grumbles * Fix failing tests * Make Account::from_rlp return Result * Syntx, sigh * Temp-fix for decoding failures * Header::decode returns Result Handle new return type throughout the code base. * Do not continue reading from the DB when a value could not be read * Fix tests * Handle header decoding in light_sync * Handling header decoding errors * Let the DecodeError bubble up unchanged * Remove redundant error conversion * Update CHANGELOG for 1.9, 1.10, and 1.11 (#8556) * Move changelog for 1.10.x * Mark 1.9 EOL * Prepare changelog for 1.10.3 stable * Prepare changelog for 1.11.0 stable * Update changelogs * Update CHANGELOG for 1.10.3 beta * Update CHANGELOG for 1.11.0 beta * Update CHANGELOG for 1.11.0 beta * Update CHANGELOG for 1.11.0 beta * Format changelog * Handle socket address parsing errors (#8545) Unpack errors and check for io::ErrorKind::InvalidInput and return our own AddressParse error. Remove the foreign link to std::net::AddrParseError and add an `impl From` for that error. Test parsing properly. * Remove unnecessary cloning in overwrite_with (#8580) * Remove unnecessary cloning in overwrite_with * Remove into_iter * changelog nit (#8585) * Rename `whisper-cli binary` to `whisper` (#8579) * rename whisper-cli binary to whisper * fix tests * Add whisper CLI to the pipelines (#8578) * Add whisper CLI to the pipelines * Address todo, ref #8579 * Added Dockerfile for alpine linux by @andresilva, closes #3565 (#8587) * Changelog and Readme (#8591) * Move changelog for 1.10.x * Mark 1.9 EOL * Prepare changelog for 1.10.3 stable * Prepare changelog for 1.11.0 stable * Update changelogs * Update CHANGELOG for 1.10.3 beta * Update CHANGELOG for 1.11.0 beta * Update CHANGELOG for 1.11.0 beta * Update CHANGELOG for 1.11.0 beta * Format changelog * Update README for 1.11 * Fix typo * Attempt to fix intermittent test failures (#8584) Occasionally should_return_correct_nonces_when_dropped_because_of_limit fails, possibly because of multiple threads competing to finish. See CI logs here for an example: https://gitlab.parity.io/parity/parity/-/jobs/86738 * Make mio optional in ethcore-io (#8537) * Make mio optional in ethcore-io * Add some annotations, plus a check for features * Increase timer for test * Fix Parity UI link (#8600) Fix link https://github.com/paritytech/parity/issues/8599 * fix compiler warning (#8590) * Block::decode() returns Result (#8586) * block_header can fail so return Result (#8581) * block_header can fail so return Result * Restore previous return type based on feedback * Fix failing doc tests running on non-code * Remove inject.js server-side injection for dapps (#8539) * Remove inject.js server-side injection for dapps * Remove dapps test `should_inject_js` Parity doesn't inject a