diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index c906ec1d26611..662e7b48cb810 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -12,7 +12,7 @@ homepage = "https://www.parity.io/" description = "Generate an API for interacting with a substrate node from FRAME metadata" [dependencies] -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] } darling = "0.14.0" frame-metadata = "15.0.0" heck = "0.4.0" @@ -20,7 +20,7 @@ proc-macro2 = "1.0.24" proc-macro-error = "1.0.4" quote = "1.0.8" syn = "1.0.58" -scale-info = { version = "2.0.0", features = ["bit-vec"] } +scale-info = "2.0.0" subxt-metadata = { version = "0.25.0", path = "../metadata" } jsonrpsee = { version = "0.16.0", features = ["async-client", "client-ws-transport", "http-client"] } hex = "0.4.3" @@ -28,4 +28,5 @@ tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] } [dev-dependencies] bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } +scale-info = { version = "2.0.0", features = ["bit-vec"] } pretty_assertions = "1.0.0" diff --git a/codegen/src/api/mod.rs b/codegen/src/api/mod.rs index a35f7ecd20763..121a46703285b 100644 --- a/codegen/src/api/mod.rs +++ b/codegen/src/api/mod.rs @@ -161,11 +161,11 @@ impl RuntimeGenerator { let mut type_substitutes = [ ( "bitvec::order::Lsb0", - parse_quote!(#crate_path::ext::bitvec::order::Lsb0), + parse_quote!(#crate_path::utils::bits::Lsb0), ), ( "bitvec::order::Msb0", - parse_quote!(#crate_path::ext::bitvec::order::Msb0), + parse_quote!(#crate_path::utils::bits::Msb0), ), ( "sp_core::crypto::AccountId32", diff --git a/codegen/src/types/tests.rs b/codegen/src/types/tests.rs index c2949d476f2b2..6c183c52167a8 100644 --- a/codegen/src/types/tests.rs +++ b/codegen/src/types/tests.rs @@ -745,10 +745,22 @@ fn generate_bitvec() { registry.register_type(&meta_type::()); let portable_types: PortableRegistry = registry.into(); + let substitutes = [ + ( + String::from("bitvec::order::Lsb0"), + parse_quote!(::subxt_path::utils::bits::Lsb0), + ), + ( + String::from("bitvec::order::Msb0"), + parse_quote!(::subxt_path::utils::bits::Msb0), + ), + ] + .into(); + let type_gen = TypeGenerator::new( &portable_types, "root", - Default::default(), + substitutes, DerivesRegistry::new(&"::subxt_path".into()), "::subxt_path".into(), ); @@ -762,8 +774,8 @@ fn generate_bitvec() { use super::root; #[derive(::subxt_path::ext::codec::Decode, ::subxt_path::ext::codec::Encode, Debug)] pub struct S { - pub lsb: ::subxt_path::ext::bitvec::vec::BitVec<::core::primitive::u8, root::bitvec::order::Lsb0>, - pub msb: ::subxt_path::ext::bitvec::vec::BitVec<::core::primitive::u16, root::bitvec::order::Msb0>, + pub lsb: ::subxt_path::utils::bits::DecodedBits<::core::primitive::u8, ::subxt_path::utils::bits::Lsb0>, + pub msb: ::subxt_path::utils::bits::DecodedBits<::core::primitive::u16, ::subxt_path::utils::bits::Msb0>, } } } diff --git a/codegen/src/types/type_path.rs b/codegen/src/types/type_path.rs index 4c26821e728f4..2391783243ce1 100644 --- a/codegen/src/types/type_path.rs +++ b/codegen/src/types/type_path.rs @@ -246,7 +246,7 @@ impl TypePathType { bit_store_type, crate_path, } => { - let type_path = parse_quote! { #crate_path::ext::bitvec::vec::BitVec<#bit_store_type, #bit_order_type> }; + let type_path = parse_quote! { #crate_path::utils::bits::DecodedBits<#bit_store_type, #bit_order_type> }; syn::Type::Path(type_path) } } diff --git a/subxt/Cargo.toml b/subxt/Cargo.toml index f340c01df29c3..9467578858a68 100644 --- a/subxt/Cargo.toml +++ b/subxt/Cargo.toml @@ -27,10 +27,10 @@ jsonrpsee-ws = ["jsonrpsee/async-client", "jsonrpsee/client-ws-transport"] jsonrpsee-web = ["jsonrpsee/async-wasm-client", "jsonrpsee/client-web-transport"] [dependencies] -bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] } -scale-info = { version = "2.0.0", features = ["bit-vec"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] } +scale-info = "2.0.0" scale-value = "0.6.0" +scale-bits = "0.3" scale-decode = "0.4.0" futures = { version = "0.3.13", default-features = false } hex = "0.4.3" @@ -54,4 +54,7 @@ derivative = "2.2.0" getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] +bitvec = "1" +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] } +scale-info = { version = "2.0.0", features = ["bit-vec"] } tokio = { version = "1.8", features = ["macros", "time", "rt-multi-thread"] } diff --git a/subxt/src/lib.rs b/subxt/src/lib.rs index 98660ba4dedd2..f1c41156b6998 100644 --- a/subxt/src/lib.rs +++ b/subxt/src/lib.rs @@ -177,9 +177,9 @@ pub use crate::{ /// Re-export external crates that are made use of in the subxt API. pub mod ext { - pub use bitvec; pub use codec; pub use frame_metadata; + pub use scale_bits; pub use scale_value; pub use sp_core; pub use sp_runtime; diff --git a/subxt/src/utils/bits.rs b/subxt/src/utils/bits.rs new file mode 100644 index 0000000000000..409597fad9ae9 --- /dev/null +++ b/subxt/src/utils/bits.rs @@ -0,0 +1,227 @@ +// Copyright 2019-2022 Parity Technologies (UK) Ltd. +// This file is dual-licensed as Apache-2.0 or GPL-3.0. +// see LICENSE for license details. + +//! Generic `scale_bits` over `bitvec`-like `BitOrder` and `BitFormat` types. + +use codec::{ + Compact, + Input, +}; +use scale_bits::{ + scale::format::{ + Format, + OrderFormat, + StoreFormat, + }, + Bits, +}; +use std::marker::PhantomData; + +macro_rules! store { + ($ident: ident; $(($ty: ident, $wrapped: ty)),*) => { + /// Associates `bitvec::store::BitStore` trait with corresponding, type-erased `scale_bits::StoreFormat` enum. + /// + /// Used to decode bit sequences by providing `scale_bits::StoreFormat` using + /// `bitvec`-like type type parameters. + pub trait $ident { + /// Corresponding `scale_bits::StoreFormat` value. + const FORMAT: StoreFormat; + /// Number of bits that the backing store types holds. + const BITS: u32; + } + + $( + impl $ident for $wrapped { + const FORMAT: StoreFormat = StoreFormat::$ty; + const BITS: u32 = <$wrapped>::BITS; + } + )* + }; + } + +macro_rules! order { + ($ident: ident; $($ty: ident),*) => { + /// Associates `bitvec::order::BitOrder` trait with corresponding, type-erased `scale_bits::OrderFormat` enum. + /// + /// Used to decode bit sequences in runtime by providing `scale_bits::OrderFormat` using + /// `bitvec`-like type type parameters. + pub trait $ident { + /// Corresponding `scale_bits::OrderFormat` value. + const FORMAT: OrderFormat; + } + + $( + #[doc = concat!("Type-level value that corresponds to `scale_bits::OrderFormat::", stringify!($ty), "` at run-time")] + #[doc = concat!(" and `bitvec::order::BitOrder::", stringify!($ty), "` at the type level.")] + #[derive(Clone, Debug, PartialEq, Eq)] + pub enum $ty {} + impl $ident for $ty { + const FORMAT: OrderFormat = OrderFormat::$ty; + } + )* + }; + } + +store!(BitStore; (U8, u8), (U16, u16), (U32, u32), (U64, u64)); +order!(BitOrder; Lsb0, Msb0); + +/// Constructs a run-time format parameters based on the corresponding type-level parameters. +fn bit_format() -> Format { + Format { + order: Order::FORMAT, + store: Store::FORMAT, + } +} + +/// `scale_bits::Bits` generic over the bit store (`u8`/`u16`/`u32`/`u64`) and bit order (LSB, MSB) +/// used for SCALE encoding/decoding. Uses `scale_bits::Bits`-default `u8` and LSB format underneath. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedBits( + Bits, + PhantomData, + PhantomData, +); + +impl DecodedBits { + /// Extracts the underlying `scale_bits::Bits` value. + pub fn into_bits(self) -> Bits { + self.0 + } + + /// References the underlying `scale_bits::Bits` value. + pub fn as_bits(&self) -> &Bits { + &self.0 + } +} + +impl core::iter::FromIterator + for DecodedBits +{ + fn from_iter>(iter: T) -> Self { + DecodedBits(Bits::from_iter(iter), PhantomData, PhantomData) + } +} + +impl codec::Decode for DecodedBits { + fn decode(input: &mut I) -> Result { + /// Equivalent of `BitSlice::MAX_BITS` on 32bit machine. + const ARCH32BIT_BITSLICE_MAX_BITS: u32 = 0x1fff_ffff; + + let Compact(bits) = >::decode(input)?; + // Otherwise it is impossible to store it on 32bit machine. + if bits > ARCH32BIT_BITSLICE_MAX_BITS { + return Err("Attempt to decode a BitVec with too many bits".into()) + } + // NOTE: Replace with `bits.div_ceil(Store::BITS)` if `int_roundings` is stabilised + let elements = (bits / Store::BITS) + u32::from(bits % Store::BITS != 0); + let bytes_in_elem = Store::BITS.saturating_div(u8::BITS); + let bytes_needed = (elements * bytes_in_elem) as usize; + + // NOTE: We could reduce allocations if it would be possible to directly + // decode from an `Input` type using a custom format (rather than default ) + // for the `Bits` type. + let mut storage = codec::Encode::encode(&Compact(bits)); + let prefix_len = storage.len(); + storage.reserve_exact(bytes_needed); + storage.extend(vec![0; bytes_needed]); + input.read(&mut storage[prefix_len..])?; + + let decoder = + scale_bits::decode_using_format_from(&storage, bit_format::())?; + let bits = decoder.collect::, _>>()?; + let bits = Bits::from_iter(bits); + + Ok(DecodedBits(bits, PhantomData, PhantomData)) + } +} + +impl codec::Encode for DecodedBits { + fn size_hint(&self) -> usize { + self.0.size_hint() + } + + fn encoded_size(&self) -> usize { + self.0.encoded_size() + } + + fn encode(&self) -> Vec { + scale_bits::encode_using_format(self.0.iter(), bit_format::()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use core::fmt::Debug; + + use bitvec::vec::BitVec; + use codec::Decode as _; + + // NOTE: We don't use `bitvec::order` types in our implementation, since we + // don't want to depend on `bitvec`. Rather than reimplementing the unsafe + // trait on our types here for testing purposes, we simply convert and + // delegate to `bitvec`'s own types. + trait ToBitVec { + type Order: bitvec::order::BitOrder; + } + impl ToBitVec for Lsb0 { + type Order = bitvec::order::Lsb0; + } + impl ToBitVec for Msb0 { + type Order = bitvec::order::Msb0; + } + + fn scales_like_bitvec_and_roundtrips< + 'a, + Store: BitStore + bitvec::store::BitStore + PartialEq, + Order: BitOrder + ToBitVec + Debug + PartialEq, + >( + input: impl IntoIterator, + ) where + BitVec::Order>: codec::Encode + codec::Decode, + { + let input: Vec<_> = input.into_iter().copied().collect(); + + let decoded_bits = DecodedBits::::from_iter(input.clone()); + let bitvec = BitVec::::Order>::from_iter(input); + + let decoded_bits_encoded = codec::Encode::encode(&decoded_bits); + let bitvec_encoded = codec::Encode::encode(&bitvec); + assert_eq!(decoded_bits_encoded, bitvec_encoded); + + let decoded_bits_decoded = + DecodedBits::::decode(&mut &decoded_bits_encoded[..]) + .expect("SCALE-encoding DecodedBits to roundtrip"); + let bitvec_decoded = + BitVec::::Order>::decode(&mut &bitvec_encoded[..]) + .expect("SCALE-encoding BitVec to roundtrip"); + assert_eq!(decoded_bits, decoded_bits_decoded); + assert_eq!(bitvec, bitvec_decoded); + } + + #[test] + fn decoded_bitvec_scales_and_roundtrips() { + let test_cases = [ + vec![], + vec![true], + vec![false], + vec![true, false, true], + vec![true, false, true, false, false, false, false, false, true], + [vec![true; 5], vec![false; 5], vec![true; 1], vec![false; 3]].concat(), + [vec![true; 9], vec![false; 9], vec![true; 9], vec![false; 9]].concat(), + ]; + + for test_case in &test_cases { + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + scales_like_bitvec_and_roundtrips::(test_case); + } + } +} diff --git a/subxt/src/utils.rs b/subxt/src/utils/mod.rs similarity index 99% rename from subxt/src/utils.rs rename to subxt/src/utils/mod.rs index d8bff8d7fb3ed..ec6761c1b78fa 100644 --- a/subxt/src/utils.rs +++ b/subxt/src/utils/mod.rs @@ -4,6 +4,8 @@ //! Miscellaneous utility helpers. +pub mod bits; + use codec::{ Decode, DecodeAll, diff --git a/testing/integration-tests/src/codegen/polkadot.rs b/testing/integration-tests/src/codegen/polkadot.rs index 54ec32b10598f..92d0d7570d6ce 100644 --- a/testing/integration-tests/src/codegen/polkadot.rs +++ b/testing/integration-tests/src/codegen/polkadot.rs @@ -38145,9 +38145,9 @@ pub mod api { Debug, )] pub struct AvailabilityBitfield( - pub ::subxt::ext::bitvec::vec::BitVec< + pub ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, ); #[derive( @@ -38163,9 +38163,9 @@ pub mod api { pub validity_votes: ::std::vec::Vec< runtime_types::polkadot_primitives::v2::ValidityAttestation, >, - pub validator_indices: ::subxt::ext::bitvec::vec::BitVec< + pub validator_indices: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, } #[derive( @@ -38255,13 +38255,13 @@ pub mod api { Debug, )] pub struct DisputeState<_0> { - pub validators_for: ::subxt::ext::bitvec::vec::BitVec< + pub validators_for: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, - pub validators_against: ::subxt::ext::bitvec::vec::BitVec< + pub validators_against: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, pub start: _0, pub concluded_at: ::core::option::Option<_0>, @@ -40088,13 +40088,13 @@ pub mod api { pub hash: runtime_types::polkadot_core_primitives::CandidateHash, pub descriptor: runtime_types::polkadot_primitives::v2::CandidateDescriptor<_0>, - pub availability_votes: ::subxt::ext::bitvec::vec::BitVec< + pub availability_votes: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, - pub backers: ::subxt::ext::bitvec::vec::BitVec< + pub backers: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, pub relay_parent_number: _1, pub backed_in_number: _1, @@ -40263,13 +40263,13 @@ pub mod api { Debug, )] pub struct PvfCheckActiveVoteState<_0> { - pub votes_accept: ::subxt::ext::bitvec::vec::BitVec< + pub votes_accept: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, - pub votes_reject: ::subxt::ext::bitvec::vec::BitVec< + pub votes_reject: ::subxt::utils::bits::DecodedBits< ::core::primitive::u8, - ::subxt::ext::bitvec::order::Lsb0, + ::subxt::utils::bits::Lsb0, >, pub age: _0, pub created_at: _0,