Introduce benchmarking using Criterion. (#110)

Cargo.toml

@@ -12,14 +12,19 @@ description = "The rust language implementation of Raft algorithm."
 categories = ["algorithms", "database-implementations"]
 
 [dependencies]
-log = "0.4.3"
+log = "0.4"
 protobuf = "2.0.4"
 quick-error = "1.2.2"
 rand = "0.5.4"
 fxhash = "0.2.1"
 
 [dev-dependencies]
-env_logger = "0.5.12"
+env_logger = "0.5"
+criterion = ">0.2.4"
+
+[[bench]]
+name = "benches"
+harness = false
 
 [badges]
 travis-ci = { repository = "pingcap/raft-rs" }

README.md

@@ -38,20 +38,26 @@ Using `rustup` you can get started this way:
 ```bash
 rustup override set stable
 rustup toolchain install nightly
+rustup component add clippy-review --nightly
+rustup component add rustfmt-preview
 ```
 
 In order to have your PR merged running the following must finish without error:
 
 ```bash
-cargo +nightly test --features dev
+cargo test --all && \
+cargo +nightly clippy --all && \
+cargo fmt --all -- --check
 ```
 
 You may optionally want to install `cargo-watch` to allow for automated rebuilding while editing:
 
 ```bash
-cargo watch -s "cargo check --features dev"
+cargo watch -s "cargo check"
 ```
 
+### Modifying Protobufs
+
 If proto file `eraftpb.proto` changed, run the command to regenerate `eraftpb.rs`:
 
 ```bash
@@ -60,6 +66,31 @@ protoc proto/eraftpb.proto --rust_out=src
 
 You can check `Cargo.toml` to find which version of `protobuf-codegen` is required.
 
+### Benchmarks
+
+We use [Criterion](https://github.com/japaric/criterion.rs) for benchmarking.
+
+> It's currently an ongoing effort to build an appropriate benchmarking suite. If you'd like to help out please let us know! [Interested?](https://github.com/pingcap/raft-rs/issues/109)
+
+You can run the benchmarks by installing `gnuplot` then running:
+
+```bash
+cargo bench
+```
+
+You can check `target/criterion/report/index.html` for plots and charts relating to the benchmarks.
+
+You can check the performance between two branches:
+
+```bash
+git checkout master
+cargo bench --bench benches -- --save-baseline master
+git checkout other
+cargo bench --bench benches -- --baseline master
+```
+
+This will report relative increases or decreased for each benchmark.
+
 ## Acknowledgments
 
 Thanks [etcd](https://github.com/coreos/etcd) for providing the amazing Go implementation!

benches/benches.rs

@@ -0,0 +1,28 @@
+#![allow(dead_code)] // Due to criterion we need this to avoid warnings.
+
+extern crate criterion;
+extern crate env_logger;
+extern crate raft;
+
+use criterion::Criterion;
+use std::time::Duration;
+
+mod suites;
+
+pub const DEFAULT_RAFT_SETS: [(usize, usize); 4] = [(0, 0), (3, 1), (5, 2), (7, 3)];
+
+fn main() {
+    criterion::init_logging();
+    let mut c = Criterion::default()
+        // Configure defaults before overriding with args.
+        .warm_up_time(Duration::from_millis(500))
+        .measurement_time(Duration::from_secs(1))
+        .configure_from_args();
+
+    suites::bench_raft(&mut c);
+    suites::bench_raw_node(&mut c);
+    suites::bench_progress(&mut c);
+    suites::bench_progress_set(&mut c);
+
+    c.final_summary();
+}

benches/suites/mod.rs

@@ -0,0 +1,8 @@
+mod raft;
+pub use self::raft::*;
+mod raw_node;
+pub use self::raw_node::*;
+mod progress;
+pub use self::progress::*;
+mod progress_set;
+pub use self::progress_set::*;

benches/suites/progress.rs

@@ -0,0 +1,15 @@
+use criterion::{Bencher, Criterion};
+use raft::Progress;
+
+pub fn bench_progress(c: &mut Criterion) {
+    bench_progress_default(c);
+}
+
+pub fn bench_progress_default(c: &mut Criterion) {
+    let bench = |b: &mut Bencher| {
+        // No setup.
+        b.iter(|| Progress::default());
+    };
+
+    c.bench_function("Progress::default", bench);
+}

benches/suites/progress_set.rs

@@ -0,0 +1,178 @@
+use criterion::{Bencher, Criterion};
+use raft::ProgressSet;
+use DEFAULT_RAFT_SETS;
+
+pub fn bench_progress_set(c: &mut Criterion) {
+    bench_progress_set_new(c);
+    bench_progress_set_insert_voter(c);
+    bench_progress_set_insert_learner(c);
+    bench_progress_set_promote_learner(c);
+    bench_progress_set_remove(c);
+    bench_progress_set_iter(c);
+    bench_progress_set_get(c);
+    bench_progress_set_nodes(c);
+}
+
+fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet {
+    let mut set = ProgressSet::new(voters, learners);
+    (0..voters).for_each(|id| {
+        set.insert_voter(id as u64, Default::default()).ok();
+    });
+    (voters..learners).for_each(|id| {
+        set.insert_learner(id as u64, Default::default()).ok();
+    });
+    set
+}
+
+pub fn bench_progress_set_new(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            // No setup.
+            b.iter(|| ProgressSet::new(voters, learners));
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::new ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_insert_voter(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.insert_voter(99, Default::default()).ok()
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::insert_voter ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_insert_learner(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.insert_learner(99, Default::default()).ok()
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::insert_learner ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_remove(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.remove(3)
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::remove ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_promote_learner(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let mut set = set.clone();
+                set.promote_learner(3)
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::promote ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_iter(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                let agg = set.iter().all(|_| true);
+                agg
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::iter ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_nodes(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                let agg = set.iter().all(|_| true);
+                agg
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::nodes ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_progress_set_get(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            let set = quick_progress_set(voters, learners);
+            b.iter(|| {
+                let set = set.clone();
+                {
+                    set.get(1);
+                }
+            });
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("ProgressSet::get ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}

benches/suites/raft.rs

@@ -0,0 +1,69 @@
+use criterion::{Bencher, Criterion};
+use raft::{storage::MemStorage, Config, Raft};
+use DEFAULT_RAFT_SETS;
+
+pub fn bench_raft(c: &mut Criterion) {
+    bench_raft_new(c);
+    bench_raft_campaign(c);
+}
+
+fn quick_raft(voters: usize, learners: usize) -> Raft<MemStorage> {
+    let id = 1;
+    let storage = MemStorage::default();
+    let config = Config::new(id);
+    let mut raft = Raft::new(&config, storage);
+    (0..voters).for_each(|id| {
+        raft.add_node(id as u64);
+    });
+    (voters..learners).for_each(|id| {
+        raft.add_learner(id as u64);
+    });
+    raft
+}
+
+pub fn bench_raft_new(c: &mut Criterion) {
+    let bench = |voters, learners| {
+        move |b: &mut Bencher| {
+            // No setup.
+            b.iter(|| quick_raft(voters, learners));
+        }
+    };
+
+    DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| {
+        c.bench_function(
+            &format!("Raft::new ({}, {})", voters, learners),
+            bench(*voters, *learners),
+        );
+    });
+}
+
+pub fn bench_raft_campaign(c: &mut Criterion) {
+    let bench = |voters, learners, variant| {
+        move |b: &mut Bencher| {
+            b.iter(|| {
+                // TODO: Make raft clone somehow.
+                let mut raft = quick_raft(voters, learners);
+                raft.campaign(variant)
+            })
+        }
+    };
+
+    DEFAULT_RAFT_SETS
+        .iter()
+        .skip(1)
+        .for_each(|(voters, learners)| {
+            // We don't want to make `raft::raft` public at this point.
+            let msgs = [
+                "CampaignPreElection",
+                "CampaignElection",
+                "CampaignTransfer",
+            ];
+            // Skip the first since it's 0,0
+            for msg in msgs.iter() {
+                c.bench_function(
+                    &format!("Raft::campaign ({}, {}, {})", voters, learners, msg),
+                    bench(*voters, *learners, msg.as_bytes()),
+                );
+            }
+        });
+}