Introduce benchmarking using Criterion.

Cargo.toml

@@ -18,6 +18,13 @@ quick-error = "1.2.2"
 rand = "0.5.4"
 fxhash = "0.2.1"
 
+[dev-dependencies]
+criterion = ">0.2.4"
+
+[[bench]]
+name = "benches"
+harness = false
+
 [badges]
 travis-ci = { repository = "pingcap/raft-rs" }
 

README.md

@@ -38,20 +38,26 @@ Using `rustup` you can get started this way:
 ```bash
 rustup override set stable
 rustup toolchain install nightly
+rustup component add clippy-review --nightly
+rustup component add rustfmt-preview
 ```
 
 In order to have your PR merged running the following must finish without error:
 
 ```bash
-cargo +nightly test --features dev
+cargo test --all && \
+cargo +nightly clippy --all && \
+cargo fmt --all -- --check
 ```
 
 You may optionally want to install `cargo-watch` to allow for automated rebuilding while editing:
 
 ```bash
-cargo watch -s "cargo check --features dev"
+cargo watch -s "cargo check"
 ```
 
+### Modifying Protobufs
+
 If proto file `eraftpb.proto` changed, run the command to regenerate `eraftpb.rs`:
 
 ```bash
@@ -60,6 +66,31 @@ protoc proto/eraftpb.proto --rust_out=src
 
 You can check `Cargo.toml` to find which version of `protobuf-codegen` is required.
 
+### Benchmarks
+
+We use [Criterion](https://github.com/japaric/criterion.rs) for benchmarking.
+
+> It's currently an ongoing effort to build an appropriate benchmarking suite. If you'd like to help out please let us know! [Interested?](https://github.com/pingcap/raft-rs/issues/109)
+
+You can run the benchmarks by installing `gnuplot` then running:
+
+```bash
+cargo bench
+```
+
+You can check `target/criterion/report/index.html` for plots and charts relating to the benchmarks.
+
+You can check the performance between two branches:
+
+```bash
+git checkout master
+cargo bench --bench benches -- --save-baseline master
+git checkout other
+cargo bench --bench benches -- --baseline master
+```
+
+This will report relative increases or decreased for each benchmark.
+
 ## Acknowledgments
 
 Thanks [etcd](https://github.com/coreos/etcd) for providing the amazing Go implementation!

benches/benches.rs

@@ -0,0 +1,15 @@
+#![allow(dead_code)] // Due to criterion we need this to avoid warnings.
+
+#[macro_use]
+extern crate criterion;
+extern crate raft;
+
+mod suites;
+use suites::*;
+
+criterion_main!(
+    bench_raft,
+    bench_raw_node,
+    bench_progress,
+    bench_progress_set,
+);

benches/suites/mod.rs

@@ -0,0 +1,8 @@
+mod raft;
+pub use self::raft::*;
+mod raw_node;
+pub use self::raw_node::*;
+mod progress;
+pub use self::progress::*;
+mod progress_set;
+pub use self::progress_set::*;

benches/suites/progress.rs

@@ -0,0 +1,13 @@
+use criterion::{Bencher, Criterion};
+use raft::Progress;
+
+criterion_group!(bench_progress, bench_progress_default);
+
+pub fn bench_progress_default(c: &mut Criterion) {
+    let bench = |b: &mut Bencher| {
+        // No setup.
+        b.iter(|| Progress::default());
+    };
+
+    c.bench_function("Progress::default", bench);
+}

benches/suites/progress_set.rs

@@ -0,0 +1,155 @@
+use criterion::{Bencher, Criterion};
+use raft::ProgressSet;
+
+criterion_group!(
+    bench_progress_set,
+    bench_progress_set_new,
+    bench_progress_set_insert_voter,
+    bench_progress_set_insert_learner,
+    bench_progress_set_promote_learner,
+    bench_progress_set_remove,
+    bench_progress_set_iter,
+    bench_progress_set_get,
+    bench_progress_set_nodes,
+);
+
+fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet {
+    let mut set = ProgressSet::new(voters, learners);
+    (0..voters).for_each(|id| {
+        set.insert_voter(id as u64, Default::default()).ok();
+    });
+    (voters..learners).for_each(|id| {
+        set.insert_learner(id as u64, Default::default()).ok();
+    });
+    set
+}
+
+pub fn bench_progress_set_new(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            // No setup.
+            b.iter(|| ProgressSet::new(voters, learners));
+        }
+    };
+
+    c.bench_function("ProgressSet::new (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::new (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::new (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::new (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_insert_voter(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.insert_voter(99, Default::default()).ok()
+            });
+        }
+    };
+    c.bench_function("ProgressSet::insert_voter (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::insert_voter (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::insert_voter (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::insert_voter (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_insert_learner(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.insert_learner(99, Default::default()).ok()
+            });
+        }
+    };
+    c.bench_function("ProgressSet::insert_learner (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::insert_learner (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::insert_learner (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::insert_learner (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_remove(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.remove(3)
+            });
+        }
+    };
+    c.bench_function("ProgressSet::remove (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::remove (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::remove (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::remove (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_promote_learner(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.promote_learner(3)
+            });
+        }
+    };
+    c.bench_function("ProgressSet::promote (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::promote (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::promote (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::promote (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_iter(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                let agg = set.iter().all(|_| true);
+                agg
+            });
+        }
+    };
+    c.bench_function("ProgressSet::iter (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::iter (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::iter (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::iter (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_nodes(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                let agg = set.iter().all(|_| true);
+                agg
+            });
+        }
+    };
+    c.bench_function("ProgressSet::nodes (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::nodes (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::nodes (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::nodes (7, 3)", bench(7, 3));
+}
+
+pub fn bench_progress_set_get(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                {
+                    set.get(1);
+                }
+            });
+        }
+    };
+    c.bench_function("ProgressSet::get (0, 0)", bench(0, 0));
+    c.bench_function("ProgressSet::get (3, 1)", bench(3, 1));
+    c.bench_function("ProgressSet::get (5, 2)", bench(5, 2));
+    c.bench_function("ProgressSet::get (7, 3)", bench(7, 3));
+}

benches/suites/raft.rs

@@ -0,0 +1,86 @@
+use criterion::{Bencher, Criterion};
+use raft::{storage::MemStorage, Config, Raft};
+
+criterion_group!(bench_raft, bench_raft_new, bench_raft_campaign,);
+
+fn quick_raft(voters: usize, learners: usize) -> Raft<MemStorage> {
+    let id = 1;
+    let storage = MemStorage::default();
+    let config = Config::new(id);
+    let mut raft = Raft::new(&config, storage);
+    (0..voters).for_each(|id| {
+        raft.add_node(id as u64);
+    });
+    (voters..learners).for_each(|id| {
+        raft.add_learner(id as u64);
+    });
+    raft
+}
+
+pub fn bench_raft_new(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            // No setup.
+            b.iter(|| quick_raft(voters, learners));
+        }
+    };
+
+    c.bench_function("Raft::new (0, 0)", bench(0, 0));
+    c.bench_function("Raft::new (3, 1)", bench(3, 1));
+    c.bench_function("Raft::new (5, 2)", bench(5, 2));
+    c.bench_function("Raft::new (7, 3)", bench(7, 3));
+}
+
+pub fn bench_raft_campaign(c: &mut Criterion) {
+    let bench = |voters, learners, variant| {
+        move |b: &mut Bencher| {
+            b.iter(|| {
+                // TODO: Make raft clone somehow.
+                let mut raft = quick_raft(voters, learners);
+                raft.campaign(variant)
+            })
+        }
+    };
+
+    // We don't want to make `raft::raft` public at this point.
+    let pre_election = b"CampaignPreElection";
+    let election = b"CampaignElection";
+    let transfer = b"CampaignTransfer";
+
+    c.bench_function(
+        "Raft::campaign (3, 1, pre_election)",
+        bench(3, 1, &pre_election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (3, 1, election)",
+        bench(3, 1, &election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (3, 1, transfer)",
+        bench(3, 1, &transfer[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (5, 2, pre_election)",
+        bench(5, 2, &pre_election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (5, 2, election)",
+        bench(5, 2, &election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (5, 2, transfer)",
+        bench(5, 2, &transfer[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (7, 3, pre_election)",
+        bench(7, 3, &pre_election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (7, 3, election)",
+        bench(7, 3, &election[..]),
+    );
+    c.bench_function(
+        "Raft::campaign (7, 3, transfer)",
+        bench(7, 3, &transfer[..]),
+    );
+}

benches/suites/raw_node.rs

@@ -0,0 +1,22 @@
+use criterion::{Bencher, Criterion};
+use raft::{storage::MemStorage, Config, RawNode};
+
+criterion_group!(bench_raw_node, bench_raw_node_new,);
+
+fn quick_raw_node() -> RawNode<MemStorage> {
+    let id = 1;
+    let peers = vec![];
+    let storage = MemStorage::default();
+    let config = Config::new(id);
+    let node = RawNode::new(&config, storage, peers).unwrap();
+    node
+}
+
+pub fn bench_raw_node_new(c: &mut Criterion) {
+    let bench = |b: &mut Bencher| {
+        // No setup.
+        b.iter(|| quick_raw_node());
+    };
+
+    c.bench_function("RawNode::new", bench);
+}

src/lib.rs

@@ -232,6 +232,9 @@ pub mod eraftpb;
 mod errors;
 mod log_unstable;
 mod progress;
+#[cfg(test)]
+pub mod raft;
+#[cfg(not(test))]
 mod raft;
 mod raft_log;
 pub mod raw_node;