Repeat the bench to get average values

next
Sayan Nandan 3 years ago
parent 4c47aa05ca
commit 0843552dcb

@ -25,9 +25,8 @@
*/
use crate::hoststr;
use crate::report;
use crate::sanity_test;
use crate::util::calc;
use crate::util::JSONReportBlock;
use devtimer::DevTime;
use libstress::utils::generate_random_string_vector;
use libstress::PoolConfig;
@ -38,6 +37,8 @@ use std::net::TcpStream;
/// Just a sweet `*1\n`
const SIMPLE_QUERY_SIZE: usize = 3;
const DEFAULT_REPEAT: usize = 5;
/// For a dataframe, this returns the dataframe size for array responses.
///
/// For example,
@ -118,22 +119,20 @@ pub fn runner(
per_kv_size: usize,
json_out: bool,
) {
println!("Running sanity test ...");
if !json_out {
println!("Running sanity test ...");
}
if let Err(e) = sanity_test!(host, port) {
err!(format!("Sanity test failed with error: {}", e));
}
println!("Finished sanity test");
if !json_out {
println!(
"Initializing benchmark\nConnections: {}\nQueries: {}\nData size (key+value): {} bytes",
max_connections,
max_queries,
(per_kv_size * 2), // key size + value size
);
println!("Finished sanity test. Initializing benchmark ...");
println!("Connections: {}", max_connections);
println!("Queries: {}", max_queries);
println!("Data size (key+value): {} bytes", (per_kv_size * 2));
}
let host = hoststr!(host, port);
let mut rand = thread_rng();
let mut dt = DevTime::new_complex();
let temp_table = libstress::utils::rand_alphastring(10, &mut rand);
let create_table = libsky::into_raw_query(&format!(
@ -206,63 +205,63 @@ pub fn runner(
println!("Per-packet size (UPDATE): {} bytes", update_packs[0].len());
println!("Initialization complete! Benchmark started");
}
let mut report = report::AggregatedReport::new(3, DEFAULT_REPEAT, max_queries);
for i in 0..DEFAULT_REPEAT {
let mut dt = DevTime::new_complex();
// clone in the keys
let set_packs = set_packs.clone();
let get_packs = get_packs.clone();
let update_packs = update_packs.clone();
// bench SET
let setpool = pool_config.get_pool();
dt.create_timer("SET").unwrap();
dt.start_timer("SET").unwrap();
setpool.execute_and_finish_iter(set_packs);
dt.stop_timer("SET").unwrap();
// bench SET
let setpool = pool_config.get_pool();
dt.create_timer("SET").unwrap();
dt.start_timer("SET").unwrap();
setpool.execute_and_finish_iter(set_packs);
dt.stop_timer("SET").unwrap();
// TODO: Update the getpool to use correct sizes
// bench GET
let get_response_packet_size =
calculate_monoelement_dataframe_size(per_kv_size) + SIMPLE_QUERY_SIZE;
let getpool = pool_config.with_loop_closure(move |sock: &mut TcpStream, packet: Vec<u8>| {
sock.write_all(&packet).unwrap();
// read exact for the key size
let mut v = vec![0; get_response_packet_size];
let _ = sock.read_exact(&mut v).unwrap();
});
dt.create_timer("GET").unwrap();
dt.start_timer("GET").unwrap();
getpool.execute_and_finish_iter(get_packs);
dt.stop_timer("GET").unwrap();
// TODO: Update the getpool to use correct sizes
// bench GET
let get_response_packet_size =
calculate_monoelement_dataframe_size(per_kv_size) + SIMPLE_QUERY_SIZE;
let getpool =
pool_config.with_loop_closure(move |sock: &mut TcpStream, packet: Vec<u8>| {
sock.write_all(&packet).unwrap();
// read exact for the key size
let mut v = vec![0; get_response_packet_size];
let _ = sock.read_exact(&mut v).unwrap();
});
dt.create_timer("GET").unwrap();
dt.start_timer("GET").unwrap();
getpool.execute_and_finish_iter(get_packs);
dt.stop_timer("GET").unwrap();
// bench UPDATE
let update_pool = pool_config.get_pool();
dt.create_timer("UPDATE").unwrap();
dt.start_timer("UPDATE").unwrap();
update_pool.execute_and_finish_iter(update_packs);
dt.stop_timer("UPDATE").unwrap();
// bench UPDATE
let update_pool = pool_config.get_pool();
dt.create_timer("UPDATE").unwrap();
dt.start_timer("UPDATE").unwrap();
update_pool.execute_and_finish_iter(update_packs);
dt.stop_timer("UPDATE").unwrap();
if !json_out {
println!("Benchmark completed! Removing created keys...");
}
if !json_out {
println!("Finished run: {}", i + 1);
}
// drop table
let drop_pool = pool_config.get_pool_with_workers(1);
drop_pool.execute(drop_table);
drop(drop_pool);
let mut maxpad = 0usize;
let mut data: Vec<JSONReportBlock> = dt
.iter()
.map(|(timer, time)| {
if timer.len() > maxpad {
maxpad = timer.len();
}
JSONReportBlock::new(timer, calc(max_queries, time.time_in_nanos().unwrap()))
})
.collect();
// sort alphabetically
data.sort();
// drop table
let drop_pool = pool_config.get_pool_with_workers(1);
drop_pool.execute(drop_table.clone());
drop(drop_pool);
dt.iter()
.for_each(|(name, timer)| report.insert(name, timer.time_in_nanos().unwrap()));
}
if json_out {
let serialized = serde_json::to_string(&data).unwrap();
let serialized = report.into_json();
println!("{}", serialized);
} else {
let pad = |clen: usize| " ".repeat(maxpad - clen);
println!("===========RESULTS===========");
data.into_iter().for_each(|block| {
let (report, maxpad) = report.into_sorted_stat();
let pad = |clen: usize| " ".repeat(maxpad - clen);
report.into_iter().for_each(|block| {
println!(
"{}{} {:.6}/sec",
block.get_report(),

@ -34,6 +34,7 @@
#[macro_use]
mod util;
mod benchtool;
mod report;
mod testkey;
use crate::util::DEFAULT_PACKET_SIZE;
use crate::util::DEFAULT_QUERY_COUNT;

@ -0,0 +1,147 @@
/*
* Created on Tue Aug 10 2021
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
* NoSQL database written by Sayan Nandan ("the Author") with the
* vision to provide flexibility in data modelling without compromising
* on performance, queryability or scalability.
*
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
use crate::util;
use core::cmp::Ordering;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
/// A map of reports
pub struct AggregatedReport {
map: HashMap<&'static str, Report>,
queries: usize,
cap: usize,
}
impl AggregatedReport {
/// Create a new aggregated report instance. Here:
/// - `report_count`: Is the count of benches you will be running. For example, if you
/// are testing GET and SET, this will be 2
/// - `cap`: Is the number of repeats you will be running
/// - `queries`: Is the number of queries you will run
pub fn new(report_count: usize, cap: usize, queries: usize) -> Self {
Self {
map: HashMap::with_capacity(report_count),
cap,
queries,
}
}
/// Insert a new statistic. The `name` should correspond to the bench name (for example GET)
/// while the `time` should be the time taken for that bench to complete
pub fn insert(&mut self, name: &'static str, time: u128) {
match self.map.entry(name) {
Entry::Occupied(mut oe) => oe.get_mut().times.push(time),
Entry::Vacant(ve) => {
let mut rep = Report::with_capacity(self.cap);
rep.times.push(time);
let _ = ve.insert(rep);
}
}
}
/// Returns a vector of sorted statistics (lexicographical) and the length of the longest
/// bench name. `(Vec<Stat>, longest_bench_name)`
pub fn into_sorted_stat(self) -> (Vec<Stat>, usize) {
let Self { map, queries, .. } = self;
let mut maxpad = 0usize;
let mut repvec: Vec<Stat> = map
.into_iter()
.map(|(name, report)| {
if name.len() > maxpad {
maxpad = name.len();
}
report.into_stat(queries, name)
})
.collect();
repvec.sort();
(repvec, maxpad)
}
/// Returns a minified JSON string
pub fn into_json(self) -> String {
serde_json::to_string(&self.into_sorted_stat().0).unwrap()
}
}
#[derive(Debug)]
/// A report with a collection of times
pub struct Report {
times: Vec<u128>,
}
impl Report {
/// Returns a new report with space for atleast `cap` number of times
pub fn with_capacity(cap: usize) -> Self {
Self {
times: Vec::with_capacity(cap),
}
}
/// Returns a [`Stat`] with the average time
pub fn into_stat(self, reqs: usize, name: &'static str) -> Stat {
let count = self.times.len();
let avg: u128 = self.times.into_iter().sum();
let avg = avg / count as u128;
Stat {
name,
stat: util::calc(reqs, avg),
}
}
}
#[derive(serde::Serialize, Debug)]
/// A statistic: name of the bench and the result
pub struct Stat {
name: &'static str,
stat: f64,
}
impl Stat {
/// Get a reference to the report name
pub fn get_report(&self) -> &str {
self.name
}
/// Get the statistic
pub fn get_stat(&self) -> f64 {
self.stat
}
}
impl PartialEq for Stat {
fn eq(&self, oth: &Self) -> bool {
self.name == oth.name
}
}
impl Eq for Stat {}
impl PartialOrd for Stat {
fn partial_cmp(&self, oth: &Self) -> Option<Ordering> {
self.name.partial_cmp(oth.name)
}
}
impl Ord for Stat {
fn cmp(&self, oth: &Self) -> std::cmp::Ordering {
self.name.cmp(oth.name)
}
}

@ -24,10 +24,8 @@
*
*/
use core::cmp::Ordering;
use libstress::utils::ran_string;
use rand::thread_rng;
use serde::Serialize;
use std::error::Error;
pub const DEFAULT_WORKER_COUNT: usize = 10;
@ -64,58 +62,6 @@ macro_rules! err {
}};
}
#[derive(Serialize)]
/// A `JSONReportBlock` represents a JSON object which contains the type of report
/// (for example `GET` or `SET`) and the number of such queries per second
///
/// This is an example of the object, when serialized into JSON:
/// ```json
/// {
/// "report" : "GET",
/// "stat" : 123456789.10,
/// }
/// ```
pub struct JSONReportBlock {
/// The type of benchmark
report: String,
/// The number of such queries per second
stat: f64,
}
impl JSONReportBlock {
pub fn new(report: &'static str, stat: f64) -> Self {
JSONReportBlock {
report: report.to_owned(),
stat,
}
}
pub const fn get_report(&self) -> &String {
&self.report
}
pub const fn get_stat(&self) -> f64 {
self.stat
}
}
impl PartialEq for JSONReportBlock {
fn eq(&self, oth: &Self) -> bool {
self.report == oth.report
}
}
impl Eq for JSONReportBlock {}
impl PartialOrd for JSONReportBlock {
fn partial_cmp(&self, oth: &Self) -> Option<Ordering> {
self.report.partial_cmp(&oth.report)
}
}
impl Ord for JSONReportBlock {
fn cmp(&self, oth: &Self) -> std::cmp::Ordering {
self.report.cmp(&oth.report)
}
}
/// Returns the number of queries/sec
pub fn calc(reqs: usize, time: u128) -> f64 {
reqs as f64 / (time as f64 / 1_000_000_000_f64)

Loading…
Cancel
Save