Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Commit

Permalink
update kvdb & co (#6111)
Browse files Browse the repository at this point in the history 
* toml changes

* REVERTME: patch

* adapt parachains db interface

* fix Cargo.toml patch after master rebase

* fix av-store

* fix chain-selection

* fix parachains-db?

* Revert "fix Cargo.toml patch after master rebase"

This reverts commit 3afcbf0.

* Revert "REVERTME: patch"

This reverts commit 464b717.

* Use `Ok` imported from prelude

Co-authored-by: Bastian Köcher <info@kchr.de>

* update lockfile for {"substrate"}

* Revert "update lockfile for {"substrate"}"

This reverts commit fdc623d.

* cargo update -p sp-io

Co-authored-by: Bastian Köcher <info@kchr.de>
Co-authored-by: parity-processbot <>
  • Loading branch information
@ordian @bkchr
ordian and bkchr committed Oct 5, 2022
1 parent 7870daf commit 3646202
Show file tree
Hide file tree
Showing 18 changed files with 294 additions and 323 deletions.
442 changes: 209 additions & 233 deletions Cargo.lock
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ readme = "README.md"
[dependencies]
polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] }
color-eyre = { version = "0.6.1", default-features = false }
parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }

[dev-dependencies]
assert_cmd = "2.0.4"
Expand Down
2 changes: 1 addition & 1 deletion core-primitives/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
parity-scale-codec = { version = "3.1.5", default-features = false, features = [ "derive" ] }
parity-util-mem = { version = "0.11.0", default-features = false, optional = true }
parity-util-mem = { version = "0.12.0", default-features = false, optional = true }

[features]
default = [ "std" ]
Expand Down
4 changes: 2 additions & 2 deletions node/core/approval-voting/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
lru = "0.8"
merlin = "2.0"
schnorrkel = "0.9.1"
kvdb = "0.11.0"
kvdb = "0.12.0"
derive_more = "0.99.17"
thiserror = "1.0.31"

Expand All @@ -40,5 +40,5 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
assert_matches = "1.4.0"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
4 changes: 2 additions & 2 deletions node/core/av-store/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ edition = "2021"
[dependencies]
futures = "0.3.21"
futures-timer = "3.0.2"
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
gum = { package = "tracing-gum", path = "../../gum" }
bitvec = "1.0.0"
Expand All @@ -24,7 +24,7 @@ polkadot-node-primitives = { path = "../../primitives" }
log = "0.4.17"
env_logger = "0.9.0"
assert_matches = "1.4.0"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"

sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
Expand Down
28 changes: 15 additions & 13 deletions node/core/av-store/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -792,8 +792,9 @@ fn note_block_included(
macro_rules! peek_num {
($iter:ident) => {
match $iter.peek() {
Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
None => None,
Some(Ok((k, _))) => Ok(decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b)),
Some(Err(_)) => Err($iter.next().expect("peek returned Some(Err); qed").unwrap_err()),
None => Ok(None),
}
};
}
Expand All @@ -819,10 +820,10 @@ async fn process_block_finalized<Context>(
let mut iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
.peekable();

match peek_num!(iter) {
match peek_num!(iter)? {
None => break, // end of iterator.
Some(n) => n,
}
Expand Down Expand Up @@ -867,10 +868,10 @@ async fn process_block_finalized<Context>(
let iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &end_prefix[..]))
.peekable();

let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash);
let batch = load_all_at_finalized_height(iter, batch_num, batch_finalized_hash)?;

// Now that we've iterated over the entire batch at this finalized height,
// update the meta.
Expand All @@ -890,22 +891,22 @@ async fn process_block_finalized<Context>(
// loads all candidates at the finalized height and maps them to `true` if finalized
// and `false` if unfinalized.
fn load_all_at_finalized_height(
mut iter: std::iter::Peekable<impl Iterator<Item = (Box<[u8]>, Box<[u8]>)>>,
mut iter: std::iter::Peekable<impl Iterator<Item = io::Result<util::database::DBKeyValue>>>,
block_number: BlockNumber,
finalized_hash: Hash,
) -> impl IntoIterator<Item = (CandidateHash, bool)> {
) -> io::Result<impl IntoIterator<Item = (CandidateHash, bool)>> {
// maps candidate hashes to true if finalized, false otherwise.
let mut candidates = HashMap::new();

// Load all candidates that were included at this height.
loop {
match peek_num!(iter) {
match peek_num!(iter)? {
None => break, // end of iterator.
Some(n) if n != block_number => break, // end of batch.
_ => {},
}

let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed")?;
let (_, block_hash, candidate_hash) =
decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");

Expand All @@ -916,7 +917,7 @@ fn load_all_at_finalized_height(
}
}

candidates
Ok(candidates)
}

fn update_blocks_at_finalized_height(
Expand Down Expand Up @@ -1214,9 +1215,10 @@ fn prune_all(db: &Arc<dyn Database>, config: &Config, clock: &dyn Clock) -> Resu
let mut tx = DBTransaction::new();
let iter = db
.iter_with_prefix(config.col_meta, &range_start[..])
.take_while(|(k, _)| &k[..] < &range_end[..]);
.take_while(|r| r.as_ref().map_or(true, |(k, _v)| &k[..] < &range_end[..]));

for (k, _v) in iter {
for r in iter {
let (k, _v) = r?;
tx.delete(config.col_meta, &k[..]);

let (_, candidate_hash) = match decode_pruning_key(&k[..]) {
Expand Down
4 changes: 2 additions & 2 deletions node/core/chain-selection/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ polkadot-primitives = { path = "../../../primitives" }
polkadot-node-primitives = { path = "../../primitives" }
polkadot-node-subsystem = { path = "../../subsystem" }
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
parity-scale-codec = "3.1.5"

Expand All @@ -22,4 +22,4 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
parking_lot = "0.12.0"
assert_matches = "1"
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
26 changes: 17 additions & 9 deletions node/core/chain-selection/src/db_backend/v1.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,16 +235,21 @@ impl Backend for DbBackend {
self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);

let val = stagnant_at_iter
.filter_map(|(k, v)| {
match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok()) {
(Some(at), Some(stagnant_at)) => Some((at, stagnant_at)),
_ => None,
}
.filter_map(|r| match r {
Ok((k, v)) =>
match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok())
{
(Some(at), Some(stagnant_at)) => Some(Ok((at, stagnant_at))),
_ => None,
},
Err(e) => Some(Err(e)),
})
.enumerate()
.take_while(|(idx, (at, _))| *at <= up_to.into() && *idx < max_elements)
.take_while(|(idx, r)| {
r.as_ref().map_or(true, |(at, _)| *at <= up_to.into() && *idx < max_elements)
})
.map(|(_, v)| v)
.collect::<Vec<_>>();
.collect::<Result<Vec<_>, _>>()?;

Ok(val)
}
Expand All @@ -254,10 +259,13 @@ impl Backend for DbBackend {
self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);

let val = blocks_at_height_iter
.filter_map(|(k, _)| decode_block_height_key(&k[..]))
.filter_map(|r| match r {
Ok((k, _)) => decode_block_height_key(&k[..]).map(Ok),
Err(e) => Some(Err(e)),
})
.next();

Ok(val)
val.transpose().map_err(Error::from)
}

fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
Expand Down
4 changes: 2 additions & 2 deletions node/core/dispute-coordinator/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition = "2021"
futures = "0.3.21"
gum = { package = "tracing-gum", path = "../../gum" }
parity-scale-codec = "3.1.5"
kvdb = "0.11.0"
kvdb = "0.12.0"
thiserror = "1.0.31"
lru = "0.8.0"
fatality = "0.0.6"
Expand All @@ -22,7 +22,7 @@ sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste


[dev-dependencies]
kvdb-memorydb = "0.11.0"
kvdb-memorydb = "0.12.0"
polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
Expand Down
2 changes: 1 addition & 1 deletion node/core/runtime-api/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition = "2021"
futures = "0.3.21"
gum = { package = "tracing-gum", path = "../../gum" }
memory-lru = "0.1.1"
parity-util-mem = { version = "0.11.0", default-features = false }
parity-util-mem = { version = "0.12.0", default-features = false }

sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }

Expand Down
2 changes: 1 addition & 1 deletion node/malus/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ polkadot-node-core-backing = { path = "../core/backing" }
polkadot-node-primitives = { path = "../primitives" }
polkadot-primitives = { path = "../../primitives" }
polkadot-node-core-pvf = { path = "../core/pvf" }
parity-util-mem = { version = "0.11.0", default-features = false, features = ["jemalloc-global"] }
parity-util-mem = { version = "0.12.0", default-features = false, features = ["jemalloc-global"] }
color-eyre = { version = "0.6.1", default-features = false }
assert_matches = "1.5"
async-trait = "0.1.57"
Expand Down
2 changes: 1 addition & 1 deletion node/overseer/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ polkadot-primitives = { path = "../../primitives" }
orchestra = "0.0.2"
gum = { package = "tracing-gum", path = "../gum" }
lru = "0.8"
parity-util-mem = { version = "0.11.0", default-features = false }
parity-util-mem = { version = "0.12.0", default-features = false }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
async-trait = "0.1.57"

Expand Down
4 changes: 2 additions & 2 deletions node/service/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ gum = { package = "tracing-gum", path = "../gum/" }
serde = { version = "1.0.137", features = ["derive"] }
serde_json = "1.0.81"
thiserror = "1.0.31"
kvdb = "0.11.0"
kvdb-rocksdb = { version = "0.15.2", optional = true }
kvdb = "0.12.0"
kvdb-rocksdb = { version = "0.16.0", optional = true }
parity-db = { version = "0.3.16", optional = true }
async-trait = "0.1.57"
lru = "0.8"
Expand Down
2 changes: 1 addition & 1 deletion node/service/src/parachains_db/upgrade.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> {
.to_str()
.ok_or_else(|| super::other_io_error("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(super::columns::v0::NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path)?;
let mut db = Database::open(&db_cfg, db_path)?;

db.add_column()?;
db.add_column()?;
Expand Down
6 changes: 3 additions & 3 deletions node/subsystem-util/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }

kvdb = "0.11.0"
parity-util-mem = { version = "0.11", default-features = false }
kvdb = "0.12.0"
parity-util-mem = { version = "0.12.0", default-features = false }
parity-db = { version = "0.3.13" }

[dev-dependencies]
Expand All @@ -44,5 +44,5 @@ log = "0.4.17"
polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
lazy_static = "1.4.0"
polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
kvdb-shared-tests = "0.9.0"
kvdb-shared-tests = "0.10.0"
tempfile = "3.1.0"
Loading

0 comments on commit 3646202

Please sign in to comment.