From b686bfefba9c9f18261d8cc0ff1afc055645d436 Mon Sep 17 00:00:00 2001 From: bader y Date: Wed, 20 Mar 2024 09:26:59 -0400 Subject: [PATCH] Defensive Programming in Substrate Reference Document (#2615) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _This PR is being continued from https://github.com/paritytech/polkadot-sdk/pull/2206, which was closed when the developer_hub was merged._ closes https://github.com/paritytech/polkadot-sdk-docs/issues/44 --- # Description This PR adds a reference document to the `developer-hub` crate (see https://github.com/paritytech/polkadot-sdk/pull/2102). This specific reference document covers defensive programming practices common within the context of developing a runtime with Substrate. In particular, this covers the following areas: - Default behavior of how Rust deals with numbers in general - How to deal with floating point numbers in runtime / fixed point arithmetic - How to deal with Integer overflows - General "safe math" / defensive programming practices for common pallet development scenarios - Defensive traits that exist within Substrate, i.e., `defensive_saturating_add `, `defensive_unwrap_or` - More general defensive programming examples (keep it concise) - Link to relevant examples where these practices are actually in production / being used - Unwrapping (or rather lack thereof) 101 todo -- - [x] Apply feedback from previous PR - [x] This may warrant a PR to append some of these docs to `sp_arithmetic` --------- Co-authored-by: Oliver Tale-Yazdi Co-authored-by: Gonçalo Pestana Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: Francisco Aguirre Co-authored-by: Radha <86818441+DrW3RK@users.noreply.github.com> --- Cargo.lock | 6 + docs/sdk/Cargo.toml | 7 + docs/sdk/src/guides/your_first_pallet/mod.rs | 6 +- .../reference_docs/defensive_programming.rs | 395 ++++++++++++++++++ docs/sdk/src/reference_docs/mod.rs | 3 +- .../safe_defensive_programming.rs | 1 - substrate/primitives/arithmetic/Cargo.toml | 3 + .../primitives/arithmetic/src/fixed_point.rs | 27 ++ substrate/primitives/arithmetic/src/lib.rs | 138 ++++-- .../primitives/arithmetic/src/per_things.rs | 36 ++ 10 files changed, 588 insertions(+), 34 deletions(-) create mode 100644 docs/sdk/src/reference_docs/defensive_programming.rs delete mode 100644 docs/sdk/src/reference_docs/safe_defensive_programming.rs diff --git a/Cargo.lock b/Cargo.lock index b055c79cbbf5..9e48887e17a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13376,7 +13376,9 @@ dependencies = [ "pallet-assets", "pallet-aura", "pallet-authorship", + "pallet-babe", "pallet-balances", + "pallet-broker", "pallet-collective", "pallet-default-config-example", "pallet-democracy", @@ -13385,6 +13387,7 @@ dependencies = [ "pallet-examples", "pallet-multisig", "pallet-proxy", + "pallet-referenda", "pallet-scheduler", "pallet-timestamp", "pallet-transaction-payment", @@ -13404,6 +13407,7 @@ dependencies = [ "scale-info", "simple-mermaid", "sp-api", + "sp-arithmetic", "sp-core", "sp-io", "sp-keyring", @@ -18395,6 +18399,7 @@ name = "sp-arithmetic" version = "23.0.0" dependencies = [ "criterion 0.4.0", + "docify 0.2.7", "integer-sqrt", "num-traits", "parity-scale-codec", @@ -18403,6 +18408,7 @@ dependencies = [ "scale-info", "serde", "sp-crypto-hashing", + "sp-std 14.0.0", "static_assertions", ] diff --git a/docs/sdk/Cargo.toml b/docs/sdk/Cargo.toml index 3f40a950c286..3f84d45640f9 100644 --- a/docs/sdk/Cargo.toml +++ b/docs/sdk/Cargo.toml @@ -81,6 +81,13 @@ sp-api = { path = "../../substrate/primitives/api" } sp-core = { path = "../../substrate/primitives/core" } sp-keyring = { path = "../../substrate/primitives/keyring" } sp-runtime = { path = "../../substrate/primitives/runtime" } +sp-arithmetic = { path = "../../substrate/primitives/arithmetic" } + +# Misc pallet dependencies +pallet-referenda = { path = "../../substrate/frame/referenda" } +pallet-broker = { path = "../../substrate/frame/broker" } +pallet-babe = { path = "../../substrate/frame/babe" } + sp-offchain = { path = "../../substrate/primitives/offchain" } sp-version = { path = "../../substrate/primitives/version" } diff --git a/docs/sdk/src/guides/your_first_pallet/mod.rs b/docs/sdk/src/guides/your_first_pallet/mod.rs index c633c0a69ede..c6e0dd0edf89 100644 --- a/docs/sdk/src/guides/your_first_pallet/mod.rs +++ b/docs/sdk/src/guides/your_first_pallet/mod.rs @@ -105,8 +105,8 @@ //! This macro will call `.into()` under the hood. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", transfer_better)] //! -//! Moreover, you will learn in the [Safe Defensive Programming -//! section](crate::reference_docs::safe_defensive_programming) that it is always recommended to use +//! Moreover, you will learn in the [Defensive Programming +//! section](crate::reference_docs::defensive_programming) that it is always recommended to use //! safe arithmetic operations in your runtime. By using [`frame::traits::CheckedSub`], we can not //! only take a step in that direction, but also improve the error handing and make it slightly more //! ergonomic. @@ -294,7 +294,7 @@ //! The following topics where used in this guide, but not covered in depth. It is suggested to //! study them subsequently: //! -//! - [`crate::reference_docs::safe_defensive_programming`]. +//! - [`crate::reference_docs::defensive_programming`]. //! - [`crate::reference_docs::frame_origin`]. //! - [`crate::reference_docs::frame_runtime_types`]. //! - The pallet we wrote in this guide was using `dev_mode`, learn more in diff --git a/docs/sdk/src/reference_docs/defensive_programming.rs b/docs/sdk/src/reference_docs/defensive_programming.rs new file mode 100644 index 000000000000..9828e1b50918 --- /dev/null +++ b/docs/sdk/src/reference_docs/defensive_programming.rs @@ -0,0 +1,395 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! [Defensive programming](https://en.wikipedia.org/wiki/Defensive_programming) is a design paradigm that enables a program to continue +//! running despite unexpected behavior, input, or events that may arise in runtime. +//! Usually, unforeseen circumstances may cause the program to stop or, in the Rust context, +//! panic!. Defensive practices allow for these circumstances to be accounted for ahead of time +//! and for them to be handled gracefully, which is in line with the intended fault-tolerant and +//! deterministic nature of blockchains. +//! +//! The Polkadot SDK is built to reflect these principles and to facilitate their usage accordingly. +//! +//! ## General Overview +//! +//! When developing within the context of the Substrate runtime, there is one golden rule: +//! +//! ***DO NOT PANIC***. There are some exceptions, but generally, this is the default precedent. +//! +//! > It’s important to differentiate between the runtime and node. The runtime refers to the core +//! > business logic of a Substrate-based chain, whereas the node refers to the outer client, which +//! > deals with telemetry and gossip from other nodes. For more information, read about +//! > [Substrate's node +//! > architecture](crate::reference_docs::wasm_meta_protocol#node-vs-runtime). It’s also important +//! > to note that the criticality of the node is slightly lesser +//! > than that of the runtime, which is why you may see `unwrap()` or other “non-defensive” +//! > approaches +//! in a few places of the node's code repository. +//! +//! Most of these practices fall within Rust's +//! colloquial usage of proper error propagation, handling, and arithmetic-based edge cases. +//! +//! General guidelines: +//! +//! - **Avoid writing functions that could explicitly panic,** such as directly using `unwrap()` on +//! a [`Result`], or accessing an out-of-bounds index on a collection. Safer methods to access +//! collection types, i.e., `get()` which allow defensive handling of the resulting [`Option`] are +//! recommended to be used. +//! - **It may be acceptable to use `except()`,** but only if one is completely certain (and has +//! performed a check beforehand) that a value won't panic upon unwrapping. *Even this is +//! discouraged*, however, as future changes to that function could then cause that statement to +//! panic. It is important to ensure all possible errors are propagated and handled effectively. +//! - **If a function *can* panic,** it usually is prefaced with `unchecked_` to indicate its +//! unsafety. +//! - **If you are writing a function that could panic,** [document it!](https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html#documenting-components) +//! - **Carefully handle mathematical operations.** Many seemingly, simplistic operations, such as +//! **arithmetic** in the runtime, could present a number of issues [(see more later in this +//! document)](#integer-overflow). Use checked arithmetic wherever possible. +//! +//! These guidelines could be summarized in the following example, where `bad_pop` is prone to +//! panicking, and `good_pop` allows for proper error handling to take place: +//! +//!```ignore +//! // Bad pop always requires that we return something, even if vector/array is empty. +//! fn bad_pop(v: Vec) -> T {} +//! // Good pop allows us to return None from the Option if need be. +//! fn good_pop(v: Vec) -> Option {} +//! ``` +//! +//! ### Defensive Traits +//! +//! The [`Defensive`](frame::traits::Defensive) trait provides a number of functions, all of which +//! provide an alternative to 'vanilla' Rust functions, e.g.,: +//! +//! - [`defensive_unwrap_or()`](frame::traits::Defensive::defensive_unwrap_or) instead of +//! `unwrap_or()` +//! - [`defensive_ok_or()`](frame::traits::DefensiveOption::defensive_ok_or) instead of `ok_or()` +//! +//! Defensive methods use [`debug_assertions`](https://doc.rust-lang.org/reference/conditional-compilation.html#debug_assertions), which panic in development, but in +//! production/release, they will merely log an error (i.e., `log::error`). +//! +//! The [`Defensive`](frame::traits::Defensive) trait and its various implementations can be found +//! [here](frame::traits::Defensive). +//! +//! ## Integer Overflow +//! +//! The Rust compiler prevents static overflow from happening at compile time. +//! The compiler panics in **debug** mode in the event of an integer overflow. In +//! **release** mode, it resorts to silently _wrapping_ the overflowed amount in a modular fashion +//! (from the `MAX` back to zero). +//! +//! In runtime development, we don't always have control over what is being supplied +//! as a parameter. For example, even this simple add function could present one of two outcomes +//! depending on whether it is in **release** or **debug** mode: +//! +//! ```ignore +//! fn naive_add(x: u8, y: u8) -> u8 { +//! x + y +//! } +//! ``` +//! If we passed overflow-able values at runtime, this could panic (or wrap if in release). +//! +//! ```ignore +//! naive_add(250u8, 10u8); // In debug mode, this would panic. In release, this would return 4. +//! ``` +//! +//! It is the silent portion of this behavior that presents a real issue. Such behavior should be +//! made obvious, especially in blockchain development, where unsafe arithmetic could produce +//! unexpected consequences like a user balance over or underflowing. +//! +//! Fortunately, there are ways to both represent and handle these scenarios depending on our +//! specific use case natively built into Rust and libraries like [`sp_arithmetic`]. +//! +//! ## Infallible Arithmetic +//! +//! Both Rust and Substrate provide safe ways to deal with numbers and alternatives to floating +//! point arithmetic. +//! +//! Known scenarios that could be fallible should be avoided: i.e., avoiding the possibility of +//! dividing/modulo by zero at any point should be mitigated. One should be opting for a +//! `checked_*` method to introduce safe arithmetic in their code in most cases. +//! +//! A developer should use fixed-point instead of floating-point arithmetic to mitigate the +//! potential for inaccuracy, rounding errors, or other unexpected behavior. +//! +//! - [Fixed point types](sp_arithmetic::fixed_point) and their associated usage can be found here. +//! - [PerThing](sp_arithmetic::per_things) and its associated types can be found here. +//! +//! Using floating point number types (i.e., f32. f64) in the runtime should be avoided, as a single non-deterministic result could cause chaos for blockchain consensus along with the issues above. For more on the specifics of the peculiarities of floating point calculations, [watch this video by the Computerphile](https://www.youtube.com/watch?v=PZRI1IfStY0). +//! +//! The following methods demonstrate different ways to handle numbers natively in Rust safely, +//! without fear of panic or unexpected behavior from wrapping. +//! +//! ### Checked Arithmetic +//! +//! **Checked operations** utilize an `Option` as a return type. This allows for +//! catching any unexpected behavior in the event of an overflow through simple pattern matching. +//! +//! This is an example of a valid operation: +#![doc = docify::embed!("./src/reference_docs/defensive_programming.rs", checked_add_example)] +//! +//! This is an example of an invalid operation. In this case, a simulated integer overflow, which +//! would simply result in `None`: +#![doc = docify::embed!( + "./src/reference_docs/defensive_programming.rs", + checked_add_handle_error_example +)] +//! +//! Suppose you aren’t sure which operation to use for runtime math. In that case, checked +//! operations are the safest bet, presenting two predictable (and erroring) outcomes that can be +//! handled accordingly (Some and None). +//! +//! The following conventions can be seen within the Polkadot SDK, where it is +//! handled in two ways: +//! +//! - As an [`Option`], using the `if let` / `if` or `match` +//! - As a [`Result`], via `ok_or` (or similar conversion to [`Result`] from [`Option`]) +//! +//! #### Handling via Option - More Verbose +//! +//! Because wrapped operations return `Option`, you can use a more verbose/explicit form of error +//! handling via `if` or `if let`: +#![doc = docify::embed!("./src/reference_docs/defensive_programming.rs", increase_balance)] +//! +//! Optionally, match may also be directly used in a more concise manner: +#![doc = docify::embed!("./src/reference_docs/defensive_programming.rs", increase_balance_match)] +//! +//! This is generally a useful convention for handling checked types and most types that return +//! `Option`. +//! +//! #### Handling via Result - Less Verbose +//! +//! In the Polkadot SDK codebase, checked operations are handled as a `Result` via `ok_or`. This is +//! a less verbose way of expressing the above. This usage often boils down to the developer’s +//! preference: +#![doc = docify::embed!("./src/reference_docs/defensive_programming.rs", increase_balance_result)] +//! +//! ### Saturating Operations +//! +//! Saturating a number limits it to the type’s upper or lower bound, even if the integer type +//! overflowed in runtime. For example, adding to `u32::MAX` would simply limit itself to +//! `u32::MAX`: +#![doc = docify::embed!("./src/reference_docs/defensive_programming.rs", saturated_add_example)] +//! +//! Saturating calculations can be used if one is very sure that something won't overflow, but wants +//! to avoid introducing the notion of any potential-panic or wrapping behavior. +//! +//! There is also a series of defensive alternatives via +//! [`DefensiveSaturating`](frame::traits::DefensiveSaturating), which introduces the same behavior +//! of the [`Defensive`](frame::traits::Defensive) trait, only with saturating, mathematical +//! operations: +#![doc = docify::embed!( + "./src/reference_docs/defensive_programming.rs", + saturated_defensive_example +)] +//! +//! ### Mathematical Operations in Substrate Development - Further Context +//! +//! As a recap, we covered the following concepts: +//! +//! 1. **Checked** operations - using [`Option`] or [`Result`] +//! 2. **Saturating** operations - limited to the lower and upper bounds of a number type +//! 3. **Wrapped** operations (the default) - wrap around to above or below the bounds of a type +//! +//! #### The problem with 'default' wrapped operations +//! +//! **Wrapped operations** cause the overflow to wrap around to either the maximum or minimum of +//! that type. Imagine this in the context of a blockchain, where there are account balances, voting +//! counters, nonces for transactions, and other aspects of a blockchain. +//! +//! While it may seem trivial, choosing how to handle numbers is quite important. As a thought +//! exercise, here are some scenarios of which will shed more light on when to use which. +//! +//! #### Bob's Overflowed Balance +//! +//! **Bob's** balance exceeds the `Balance` type on the `EduChain`. Because the pallet developer did +//! not handle the calculation to add to Bob's balance with any regard to this overflow, **Bob's** +//! balance is now essentially `0`, the operation **wrapped**. +//! +//!
+//! Solution: Saturating or Checked +//! For Bob's balance problems, using a `saturating_add` or `checked_add` could've mitigated +//! this issue. They simply would've reached the upper, or lower bounds, of the particular type for +//! an on-chain balance. In other words: Bob's balance would've stayed at the maximum of the +//! Balance type.
+//! +//! #### Alice's 'Underflowed' Balance +//! +//! Alice’s balance has reached `0` after a transfer to Bob. Suddenly, she has been slashed on +//! EduChain, causing her balance to reach near the limit of `u32::MAX` - a very large amount - as +//! wrapped operations can go both ways. Alice can now successfully vote using her new, overpowered +//! token balance, destroying the chain's integrity. +//! +//!
+//! Solution: Saturating +//! For Alice's balance problem, using `saturated_sub` could've mitigated this issue. A saturating +//! calculation would've simply limited her balance to the lower bound of u32, as having a negative +//! balance is not a concept within blockchains. In other words: Alice's balance would've stayed +//! at "0", even after being slashed. +//! +//! This is also an example that while one system may work in isolation, shared interfaces, such +//! as the notion of balances, are often shared across multiple pallets - meaning these small +//! changes can make a big difference depending on the scenario.
+//! +//! #### Proposal ID Overwrite +//! +//! A `u8` parameter, called `proposals_count`, represents the type for counting the number of +//! proposals on-chain. Every time a new proposal is added to the system, this number increases. +//! With the proposal pallet's high usage, it has reached `u8::MAX`’s limit of 255, causing +//! `proposals_count` to go to 0. Unfortunately, this results in new proposals overwriting old ones, +//! effectively erasing any notion of past proposals! +//! +//!
+//! Solution: Checked +//! For the proposal IDs, proper handling via `checked` math would've been suitable, +//! Saturating could've been used - but it also would've 'failed' silently. Using `checked_add` to +//! ensure that the next proposal ID would've been valid would've been a viable way to let the user +//! know the state of their proposal: +//! +//! ```ignore +//! let next_proposal_id = current_count.checked_add(1).ok_or_else(|| Error::TooManyProposals)?; +//! ``` +//! +//!
+//! +//! From the above, we can clearly see the problematic nature of seemingly simple operations in the +//! runtime, and care should be given to ensure a defensive approach is taken. +//! +//! ### Edge cases of `panic!`-able instances in Substrate +//! +//! As you traverse through the codebase (particularly in `substrate/frame`, where the majority of +//! runtime code lives), you may notice that there (only a few!) occurrences where `panic!` is used +//! explicitly. This is used when the runtime should stall, rather than keep running, as that is +//! considered safer. Particularly when it comes to mission-critical components, such as block +//! authoring, consensus, or other protocol-level dependencies, going through with an action may +//! actually cause harm to the network, and thus stalling would be the better option. +//! +//! Take the example of the BABE pallet ([`pallet_babe`]), which doesn't allow for a validator to +//! participate if it is disabled (see: [`frame::traits::DisabledValidators`]): +//! +//! ```ignore +//! if T::DisabledValidators::is_disabled(authority_index) { +//! panic!( +//! "Validator with index {:?} is disabled and should not be attempting to author blocks.", +//! authority_index, +//! ); +//! } +//! ``` +//! +//! There are other examples in various pallets, mostly those crucial to the blockchain’s +//! functionality. Most of the time, you will not be writing pallets which operate at this level, +//! but these exceptions should be noted regardless. +//! +//! ## Other Resources +//! +//! - [PBA Book - FRAME Tips & Tricks](https://polkadot-blockchain-academy.github.io/pba-book/substrate/tips-tricks/page.html?highlight=perthing#substrate-and-frame-tips-and-tricks) +#![allow(dead_code)] +#[allow(unused_variables)] +mod fake_runtime_types { + // Note: The following types are purely for the purpose of example, and do not contain any + // *real* use case other than demonstrating various concepts. + pub enum RuntimeError { + Overflow, + UserDoesntExist, + } + + pub type Address = (); + + pub struct Runtime; + + impl Runtime { + fn get_balance(account: Address) -> Result { + Ok(0u64) + } + + fn set_balance(account: Address, new_balance: u64) {} + } + + #[docify::export] + fn increase_balance(account: Address, amount: u64) -> Result<(), RuntimeError> { + // Get a user's current balance + let balance = Runtime::get_balance(account)?; + // SAFELY increase the balance by some amount + if let Some(new_balance) = balance.checked_add(amount) { + Runtime::set_balance(account, new_balance); + Ok(()) + } else { + Err(RuntimeError::Overflow) + } + } + + #[docify::export] + fn increase_balance_match(account: Address, amount: u64) -> Result<(), RuntimeError> { + // Get a user's current balance + let balance = Runtime::get_balance(account)?; + // SAFELY increase the balance by some amount + let new_balance = match balance.checked_add(amount) { + Some(balance) => balance, + None => { + return Err(RuntimeError::Overflow); + }, + }; + Runtime::set_balance(account, new_balance); + Ok(()) + } + + #[docify::export] + fn increase_balance_result(account: Address, amount: u64) -> Result<(), RuntimeError> { + // Get a user's current balance + let balance = Runtime::get_balance(account)?; + // SAFELY increase the balance by some amount - this time, by using `ok_or` + let new_balance = balance.checked_add(amount).ok_or(RuntimeError::Overflow)?; + Runtime::set_balance(account, new_balance); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use frame::traits::DefensiveSaturating; + #[docify::export] + #[test] + fn checked_add_example() { + // This is valid, as 20 is perfectly within the bounds of u32. + let add = (10u32).checked_add(10); + assert_eq!(add, Some(20)) + } + + #[docify::export] + #[test] + fn checked_add_handle_error_example() { + // This is invalid - we are adding something to the max of u32::MAX, which would overflow. + // Luckily, checked_add just marks this as None! + let add = u32::MAX.checked_add(10); + assert_eq!(add, None) + } + + #[docify::export] + #[test] + fn saturated_add_example() { + // Saturating add simply saturates + // to the numeric bound of that type if it overflows. + let add = u32::MAX.saturating_add(10); + assert_eq!(add, u32::MAX) + } + + #[docify::export] + #[test] + #[cfg_attr(debug_assertions, should_panic(expected = "Defensive failure has been triggered!"))] + fn saturated_defensive_example() { + let saturated_defensive = u32::MAX.defensive_saturating_add(10); + assert_eq!(saturated_defensive, u32::MAX); + } +} diff --git a/docs/sdk/src/reference_docs/mod.rs b/docs/sdk/src/reference_docs/mod.rs index de0b012bb126..a0d8d05b4492 100644 --- a/docs/sdk/src/reference_docs/mod.rs +++ b/docs/sdk/src/reference_docs/mod.rs @@ -47,8 +47,7 @@ pub mod signed_extensions; pub mod frame_origin; /// Learn about how to write safe and defensive code in your FRAME runtime. -// TODO: @CrackTheCode016 https://github.com/paritytech/polkadot-sdk-docs/issues/44 -pub mod safe_defensive_programming; +pub mod defensive_programming; /// Learn about composite enums and other runtime level types, such as "RuntimeEvent" and /// "RuntimeCall". diff --git a/docs/sdk/src/reference_docs/safe_defensive_programming.rs b/docs/sdk/src/reference_docs/safe_defensive_programming.rs deleted file mode 100644 index 9d0f028e570d..000000000000 --- a/docs/sdk/src/reference_docs/safe_defensive_programming.rs +++ /dev/null @@ -1 +0,0 @@ -//! diff --git a/substrate/primitives/arithmetic/Cargo.toml b/substrate/primitives/arithmetic/Cargo.toml index 29c406b10b7f..64c1025b5858 100644 --- a/substrate/primitives/arithmetic/Cargo.toml +++ b/substrate/primitives/arithmetic/Cargo.toml @@ -26,6 +26,8 @@ num-traits = { version = "0.2.17", default-features = false } scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } serde = { features = ["alloc", "derive"], optional = true, workspace = true } static_assertions = "1.1.0" +sp-std = { path = "../std", default-features = false } +docify = "0.2.7" [dev-dependencies] criterion = "0.4.0" @@ -41,6 +43,7 @@ std = [ "scale-info/std", "serde/std", "sp-crypto-hashing/std", + "sp-std/std", ] # Serde support without relying on std features. serde = ["dep:serde", "scale-info/serde"] diff --git a/substrate/primitives/arithmetic/src/fixed_point.rs b/substrate/primitives/arithmetic/src/fixed_point.rs index 46c09df21868..736a900bde24 100644 --- a/substrate/primitives/arithmetic/src/fixed_point.rs +++ b/substrate/primitives/arithmetic/src/fixed_point.rs @@ -16,6 +16,33 @@ // limitations under the License. //! Decimal Fixed Point implementations for Substrate runtime. +//! Similar to types that implement [`PerThing`](crate::per_things), these are also +//! fixed-point types, however, they are able to represent larger fractions: +#![doc = docify::embed!("./src/lib.rs", fixed_u64)] +//! +//! ### Fixed Point Types in Practice +//! +//! If one needs to exceed the value of one (1), then +//! [`FixedU64`](FixedU64) (and its signed and `u128` counterparts) can be utilized. +//! Take for example this very rudimentary pricing mechanism, where we wish to calculate the demand +//! / supply to get a price for some on-chain compute: +#![doc = docify::embed!( + "./src/lib.rs", + fixed_u64_block_computation_example +)] +//! +//! For a much more comprehensive example, be sure to look at the source for broker (the "coretime") +//! pallet. +//! +//! #### Fixed Point Types in Practice +//! +//! Just as with [`PerThing`](PerThing), you can also perform regular mathematical +//! expressions: +#![doc = docify::embed!( + "./src/lib.rs", + fixed_u64_operation_example +)] +//! use crate::{ helpers_128bit::{multiply_by_rational_with_rounding, sqrt}, diff --git a/substrate/primitives/arithmetic/src/lib.rs b/substrate/primitives/arithmetic/src/lib.rs index 33992e15423f..01c403a7c4af 100644 --- a/substrate/primitives/arithmetic/src/lib.rs +++ b/substrate/primitives/arithmetic/src/lib.rs @@ -101,7 +101,7 @@ where fn tcmp(&self, other: &T, threshold: T) -> Ordering { // early exit. if threshold.is_zero() { - return self.cmp(other) + return self.cmp(other); } let upper_bound = other.saturating_add(threshold); @@ -206,12 +206,12 @@ where // Nothing to do here. if count.is_zero() { - return Ok(Vec::::new()) + return Ok(Vec::::new()); } let diff = targeted_sum.max(sum) - targeted_sum.min(sum); if diff.is_zero() { - return Ok(input.to_vec()) + return Ok(input.to_vec()); } let needs_bump = targeted_sum > sum; @@ -254,7 +254,7 @@ where min_index += 1; min_index %= count; } - leftover -= One::one() + leftover -= One::one(); } } else { // must decrease the stakes a bit. decrement from the max element. index of maximum is now @@ -288,7 +288,7 @@ where if output_with_idx[max_index].1 <= threshold { max_index = max_index.checked_sub(1).unwrap_or(count - 1); } - leftover -= One::one() + leftover -= One::one(); } else { max_index = max_index.checked_sub(1).unwrap_or(count - 1); } @@ -300,7 +300,7 @@ where targeted_sum, "sum({:?}) != {:?}", output_with_idx, - targeted_sum, + targeted_sum ); // sort again based on the original index. @@ -356,7 +356,7 @@ mod normalize_tests { vec![ Perbill::from_parts(333333334), Perbill::from_parts(333333333), - Perbill::from_parts(333333333), + Perbill::from_parts(333333333) ] ); @@ -367,7 +367,7 @@ mod normalize_tests { vec![ Perbill::from_parts(316666668), Perbill::from_parts(383333332), - Perbill::from_parts(300000000), + Perbill::from_parts(300000000) ] ); } @@ -378,13 +378,13 @@ mod normalize_tests { // could have a situation where the sum cannot be calculated in the inner type. Calculating // using the upper type of the per_thing should assure this to be okay. assert_eq!( - vec![PerU16::from_percent(40), PerU16::from_percent(40), PerU16::from_percent(40),] + vec![PerU16::from_percent(40), PerU16::from_percent(40), PerU16::from_percent(40)] .normalize(PerU16::one()) .unwrap(), vec![ PerU16::from_parts(21845), // 33% PerU16::from_parts(21845), // 33% - PerU16::from_parts(21845), // 33% + PerU16::from_parts(21845) // 33% ] ); } @@ -428,6 +428,88 @@ mod normalize_tests { } } +#[cfg(test)] +mod per_and_fixed_examples { + use super::*; + + #[docify::export] + #[test] + fn percent_mult() { + let percent = Percent::from_rational(5u32, 100u32); // aka, 5% + let five_percent_of_100 = percent * 100u32; // 5% of 100 is 5. + assert_eq!(five_percent_of_100, 5) + } + #[docify::export] + #[test] + fn perbill_example() { + let p = Perbill::from_percent(80); + // 800000000 bil, or a representative of 0.800000000. + // Precision is in the billions place. + assert_eq!(p.deconstruct(), 800000000); + } + + #[docify::export] + #[test] + fn percent_example() { + let percent = Percent::from_rational(190u32, 400u32); + assert_eq!(percent.deconstruct(), 47); + } + + #[docify::export] + #[test] + fn fixed_u64_block_computation_example() { + // Calculate a very rudimentary on-chain price from supply / demand + // Supply: Cores available per block + // Demand: Cores being ordered per block + let price = FixedU64::from_rational(5u128, 10u128); + + // 0.5 DOT per core + assert_eq!(price, FixedU64::from_float(0.5)); + + // Now, the story has changed - lots of demand means we buy as many cores as there + // available. This also means that price goes up! For the sake of simplicity, we don't care + // about who gets a core - just about our very simple price model + + // Calculate a very rudimentary on-chain price from supply / demand + // Supply: Cores available per block + // Demand: Cores being ordered per block + let price = FixedU64::from_rational(19u128, 10u128); + + // 1.9 DOT per core + assert_eq!(price, FixedU64::from_float(1.9)); + } + + #[docify::export] + #[test] + fn fixed_u64() { + // The difference between this and perthings is perthings operates within the relam of [0, + // 1] In cases where we need > 1, we can used fixed types such as FixedU64 + + let rational_1 = FixedU64::from_rational(10, 5); //" 200%" aka 2. + let rational_2 = FixedU64::from_rational_with_rounding(5, 10, Rounding::Down); // "50%" aka 0.50... + + assert_eq!(rational_1, (2u64).into()); + assert_eq!(rational_2.into_perbill(), Perbill::from_float(0.5)); + } + + #[docify::export] + #[test] + fn fixed_u64_operation_example() { + let rational_1 = FixedU64::from_rational(10, 5); // "200%" aka 2. + let rational_2 = FixedU64::from_rational(8, 5); // "160%" aka 1.6. + + let addition = rational_1 + rational_2; + let multiplication = rational_1 * rational_2; + let division = rational_1 / rational_2; + let subtraction = rational_1 - rational_2; + + assert_eq!(addition, FixedU64::from_float(3.6)); + assert_eq!(multiplication, FixedU64::from_float(3.2)); + assert_eq!(division, FixedU64::from_float(1.25)); + assert_eq!(subtraction, FixedU64::from_float(0.4)); + } +} + #[cfg(test)] mod threshold_compare_tests { use super::*; @@ -440,15 +522,15 @@ mod threshold_compare_tests { let e = Perbill::from_percent(10).mul_ceil(b); // [115 - 11,5 (103,5), 115 + 11,5 (126,5)] is all equal - assert_eq!(103u32.tcmp(&b, e), Ordering::Equal); - assert_eq!(104u32.tcmp(&b, e), Ordering::Equal); - assert_eq!(115u32.tcmp(&b, e), Ordering::Equal); - assert_eq!(120u32.tcmp(&b, e), Ordering::Equal); - assert_eq!(126u32.tcmp(&b, e), Ordering::Equal); - assert_eq!(127u32.tcmp(&b, e), Ordering::Equal); - - assert_eq!(128u32.tcmp(&b, e), Ordering::Greater); - assert_eq!(102u32.tcmp(&b, e), Ordering::Less); + assert_eq!((103u32).tcmp(&b, e), Ordering::Equal); + assert_eq!((104u32).tcmp(&b, e), Ordering::Equal); + assert_eq!((115u32).tcmp(&b, e), Ordering::Equal); + assert_eq!((120u32).tcmp(&b, e), Ordering::Equal); + assert_eq!((126u32).tcmp(&b, e), Ordering::Equal); + assert_eq!((127u32).tcmp(&b, e), Ordering::Equal); + + assert_eq!((128u32).tcmp(&b, e), Ordering::Greater); + assert_eq!((102u32).tcmp(&b, e), Ordering::Less); } #[test] @@ -458,15 +540,15 @@ mod threshold_compare_tests { let e = Perbill::from_parts(100) * b; // [115 - 11,5 (103,5), 115 + 11,5 (126,5)] is all equal - assert_eq!(103u32.tcmp(&b, e), 103u32.cmp(&b)); - assert_eq!(104u32.tcmp(&b, e), 104u32.cmp(&b)); - assert_eq!(115u32.tcmp(&b, e), 115u32.cmp(&b)); - assert_eq!(120u32.tcmp(&b, e), 120u32.cmp(&b)); - assert_eq!(126u32.tcmp(&b, e), 126u32.cmp(&b)); - assert_eq!(127u32.tcmp(&b, e), 127u32.cmp(&b)); - - assert_eq!(128u32.tcmp(&b, e), 128u32.cmp(&b)); - assert_eq!(102u32.tcmp(&b, e), 102u32.cmp(&b)); + assert_eq!((103u32).tcmp(&b, e), (103u32).cmp(&b)); + assert_eq!((104u32).tcmp(&b, e), (104u32).cmp(&b)); + assert_eq!((115u32).tcmp(&b, e), (115u32).cmp(&b)); + assert_eq!((120u32).tcmp(&b, e), (120u32).cmp(&b)); + assert_eq!((126u32).tcmp(&b, e), (126u32).cmp(&b)); + assert_eq!((127u32).tcmp(&b, e), (127u32).cmp(&b)); + + assert_eq!((128u32).tcmp(&b, e), (128u32).cmp(&b)); + assert_eq!((102u32).tcmp(&b, e), (102u32).cmp(&b)); } #[test] diff --git a/substrate/primitives/arithmetic/src/per_things.rs b/substrate/primitives/arithmetic/src/per_things.rs index 057bfd7bf885..f73dbe30cec1 100644 --- a/substrate/primitives/arithmetic/src/per_things.rs +++ b/substrate/primitives/arithmetic/src/per_things.rs @@ -15,6 +15,42 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Types that implement [`PerThing`](PerThing) can be used as a floating-point alternative for +//! numbers that operate within the realm of `[0, 1]`. The primary types may you encounter in +//! Substrate would be the following: +//! - [`Percent`](Percent) - parts of one hundred. +//! - [`Permill`](Permill) - parts of a million. +//! - [`Perbill`](Perbill) - parts of a billion. +//! +//! In use, you may see them being used as follows: +//! +//! > **[`Perbill`](Perbill), parts of a billion** +#![doc = docify::embed!("./src/lib.rs", perbill_example)] +//! > **[`Percent`](Percent), parts of a hundred** +#![doc = docify::embed!("./src/lib.rs", percent_example)] +//! +//! Note that `Percent` is represented as a _rounded down_, fixed point +//! number (see the example above). Unlike primitive types, types that implement +//! [`PerThing`](PerThing) will also not overflow, and are therefore safe to use. +//! They adopt the same behavior that a saturated calculation would provide, meaning that if one is +//! to go over "100%", it wouldn't overflow, but simply stop at the upper or lower bound. +//! +//! For use cases which require precision beyond the range of `[0, 1]`, there are fixed-point types +//! which can be used. +//! +//! Each of these can be used to construct and represent ratios within our runtime. +//! You will find types like [`Perbill`](Perbill) being used often in pallet +//! development. `pallet_referenda` is a good example of a pallet which makes good use of fixed +//! point arithmetic, as it relies on representing various curves and thresholds relating to +//! governance. +//! +//! #### Fixed Point Arithmetic with [`PerThing`](PerThing) +//! +//! As stated, one can also perform mathematics using these types directly. For example, finding the +//! percentage of a particular item: + +#![doc = docify::embed!("./src/lib.rs", percent_mult)] + #[cfg(feature = "serde")] use serde::{Deserialize, Serialize};