Remove old tools
Sayan Nandan
10 months ago
parent
0fed52fe8a
commit
fff5780586
@ -1,9 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
noart = true
|
|
||||||
|
|
||||||
[ssl]
|
|
||||||
key="../key.pem"
|
|
||||||
chain="../cert.pem"
|
|
||||||
port = 2004
|
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
system:
|
||||||
|
mode: prod
|
||||||
|
|
||||||
|
auth:
|
||||||
|
plugin: pwd
|
||||||
|
root_pass: password12345678
|
||||||
|
|
||||||
|
endpoints:
|
||||||
|
insecure:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 2003
|
||||||
|
secure:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 2004
|
||||||
|
cert: ../cert.pem
|
||||||
|
private_key: ../key.pem
|
||||||
|
pkey_passphrase: ../passphrase.txt
|
||||||
@ -1,12 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2005
|
|
||||||
noart = true
|
|
||||||
|
|
||||||
[auth]
|
|
||||||
origin_key = "4527387f92a381cbe804593f33991d327d456a97"
|
|
||||||
|
|
||||||
[ssl]
|
|
||||||
key = "../key.pem"
|
|
||||||
chain = "../cert.pem"
|
|
||||||
port = 2006
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2007
|
|
||||||
noart = true
|
|
||||||
|
|
||||||
[snapshot]
|
|
||||||
every = 3600
|
|
||||||
atmost = 4
|
|
||||||
failsafe = true
|
|
||||||
|
|
||||||
[ssl]
|
|
||||||
key = "../key.pem"
|
|
||||||
chain = "../cert.pem"
|
|
||||||
port = 2008
|
|
||||||
@ -1,55 +0,0 @@
|
|||||||
#
|
|
||||||
# Created on Tue Nov 03 2020
|
|
||||||
#
|
|
||||||
# This file is a part of Skytable
|
|
||||||
# Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
#
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Affero General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
|
||||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
#
|
|
||||||
|
|
||||||
name: Skytable Shell
|
|
||||||
version: 0.8.0
|
|
||||||
author: Sayan N. <ohsayan@outlook.com>
|
|
||||||
about: The Skytable Shell (skysh)
|
|
||||||
args:
|
|
||||||
- host:
|
|
||||||
short: h
|
|
||||||
required: false
|
|
||||||
long: host
|
|
||||||
value_name: host
|
|
||||||
help: Sets the remote host to connect to
|
|
||||||
takes_value: true
|
|
||||||
- port:
|
|
||||||
short: p
|
|
||||||
required: false
|
|
||||||
long: port
|
|
||||||
value_name: port
|
|
||||||
help: Sets the remote port to connect to
|
|
||||||
takes_value: true
|
|
||||||
- eval:
|
|
||||||
short: e
|
|
||||||
required: false
|
|
||||||
long: eval
|
|
||||||
multiple: true
|
|
||||||
value_name: expression
|
|
||||||
help: Run an expression without REPL
|
|
||||||
takes_value: true
|
|
||||||
- cert:
|
|
||||||
short: C
|
|
||||||
required: false
|
|
||||||
long: sslcert
|
|
||||||
value_name: cert
|
|
||||||
help: Sets the PEM certificate to use for SSL connections
|
|
||||||
takes_value: true
|
|
||||||
@ -1,122 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Wed Nov 03 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
macro_rules! write_str {
|
|
||||||
($st:ident) => {
|
|
||||||
println!("\"{}\"", $st)
|
|
||||||
};
|
|
||||||
($idx:ident, $st:ident) => {
|
|
||||||
println!("({}) \"{}\"", $idx, $st)
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! write_binstr {
|
|
||||||
($st:ident) => {
|
|
||||||
println!("{}", BinaryData($st))
|
|
||||||
};
|
|
||||||
($idx:ident, $st:ident) => {
|
|
||||||
println!("({}) {}", $idx, BinaryData($st))
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! write_int {
|
|
||||||
($int:ident) => {
|
|
||||||
println!("{}", $int)
|
|
||||||
};
|
|
||||||
($idx:ident, $st:ident) => {
|
|
||||||
println!("({}) \"{}\"", $idx, $st)
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! write_err {
|
|
||||||
($idx:expr, $err:ident) => {
|
|
||||||
err!(if let Some(idx) = $idx {
|
|
||||||
format!("({}) ({})\n", idx, $err)
|
|
||||||
} else {
|
|
||||||
format!("({})\n", $err)
|
|
||||||
})
|
|
||||||
};
|
|
||||||
($idx:ident, $err:literal) => {
|
|
||||||
err!(
|
|
||||||
(if let Some(idx) = $idx {
|
|
||||||
format!("({}) ({})\n", idx, $err)
|
|
||||||
} else {
|
|
||||||
format!("({})\n", $err)
|
|
||||||
})
|
|
||||||
)
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! write_okay {
|
|
||||||
() => {
|
|
||||||
crossterm::execute!(
|
|
||||||
std::io::stdout(),
|
|
||||||
SetForegroundColor(Color::Cyan),
|
|
||||||
Print("(Okay)\n".to_string()),
|
|
||||||
ResetColor
|
|
||||||
)
|
|
||||||
.expect("Failed to write to stdout")
|
|
||||||
};
|
|
||||||
($idx:ident) => {
|
|
||||||
crossterm::execute!(
|
|
||||||
std::io::stdout(),
|
|
||||||
SetForegroundColor(Color::Cyan),
|
|
||||||
Print(format!("({}) (Okay)\n", $idx)),
|
|
||||||
ResetColor
|
|
||||||
)
|
|
||||||
.expect("Failed to write to stdout")
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! err {
|
|
||||||
($input:expr) => {
|
|
||||||
crossterm::execute!(
|
|
||||||
std::io::stdout(),
|
|
||||||
::crossterm::style::SetForegroundColor(::crossterm::style::Color::Red),
|
|
||||||
::crossterm::style::Print($input),
|
|
||||||
::crossterm::style::ResetColor
|
|
||||||
)
|
|
||||||
.expect("Failed to write to stdout")
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! eskysh {
|
|
||||||
($e:expr) => {
|
|
||||||
err!(format!("[SKYSH ERROR] {}\n", $e))
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! fatal {
|
|
||||||
($e:expr) => {{
|
|
||||||
err!($e);
|
|
||||||
::std::process::exit(0x01);
|
|
||||||
}};
|
|
||||||
($e:expr, $desc:expr) => {{
|
|
||||||
err!(format!($e, $desc));
|
|
||||||
println!();
|
|
||||||
::std::process::exit(0x01)
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
@ -1,262 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Wed May 12 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
crate::tokenizer,
|
|
||||||
core::fmt,
|
|
||||||
crossterm::style::{Color, Print, ResetColor, SetForegroundColor},
|
|
||||||
skytable::{
|
|
||||||
aio, error::Error, types::Array, types::FlatElement, Element, Pipeline, Query, RespCode,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
type SkyResult<T> = Result<T, Error>;
|
|
||||||
|
|
||||||
pub enum Runner {
|
|
||||||
Insecure(aio::Connection),
|
|
||||||
Secure(aio::TlsConnection),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Runner {
|
|
||||||
pub async fn new_insecure(host: &str, port: u16) -> SkyResult<Self> {
|
|
||||||
let con = aio::Connection::new(host, port).await?;
|
|
||||||
Ok(Self::Insecure(con))
|
|
||||||
}
|
|
||||||
pub async fn new_secure(host: &str, port: u16, cert: &str) -> SkyResult<Self> {
|
|
||||||
let con = aio::TlsConnection::new(host, port, cert).await?;
|
|
||||||
Ok(Self::Secure(con))
|
|
||||||
}
|
|
||||||
pub async fn run_pipeline(&mut self, pipeline: Pipeline) {
|
|
||||||
let ret = match self {
|
|
||||||
Self::Insecure(con) => con.run_pipeline(pipeline).await,
|
|
||||||
Self::Secure(con) => con.run_pipeline(pipeline).await,
|
|
||||||
};
|
|
||||||
let retok = match ret {
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(e) => fatal!("An I/O error occurred while querying: {}", e),
|
|
||||||
};
|
|
||||||
for (idx, resp) in retok
|
|
||||||
.into_iter()
|
|
||||||
.enumerate()
|
|
||||||
.map(|(idx, resp)| (idx + 1, resp))
|
|
||||||
{
|
|
||||||
println!("[Response {}]", idx);
|
|
||||||
print_element(resp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub async fn run_query(&mut self, unescaped: &str) {
|
|
||||||
let query: Query = match tokenizer::get_query(unescaped.as_bytes()) {
|
|
||||||
Ok(q) => q,
|
|
||||||
Err(e) => {
|
|
||||||
err!(format!("[Syntax Error: {}]\n", e));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let ret = match self {
|
|
||||||
Self::Insecure(con) => con.run_query_raw(&query).await,
|
|
||||||
Self::Secure(con) => con.run_query_raw(&query).await,
|
|
||||||
};
|
|
||||||
match ret {
|
|
||||||
Ok(resp) => print_element(resp),
|
|
||||||
Err(e) => fatal!("An I/O error occurred while querying: {}", e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub async fn check_entity(&mut self, blank: &mut String, prompt: &mut String) {
|
|
||||||
let query: Query = tokenizer::get_query(b"whereami").unwrap();
|
|
||||||
let ret = match self {
|
|
||||||
Self::Insecure(con) => con.run_query_raw(&query).await,
|
|
||||||
Self::Secure(con) => con.run_query_raw(&query).await,
|
|
||||||
};
|
|
||||||
let ret = match ret {
|
|
||||||
Ok(resp) => resp,
|
|
||||||
Err(e) => fatal!("An I/O error occurred while querying: {}", e),
|
|
||||||
};
|
|
||||||
match ret {
|
|
||||||
Element::Array(Array::NonNullStr(srr)) => match srr.len() {
|
|
||||||
1 => {
|
|
||||||
*blank = format!(" {blank}> ", blank = " ".repeat(srr[0].len()));
|
|
||||||
*prompt = format!("skysh@{ks}> ", ks = srr[0]);
|
|
||||||
}
|
|
||||||
2 => {
|
|
||||||
let ks = &srr[0];
|
|
||||||
let tbl = &srr[1];
|
|
||||||
*blank = format!(
|
|
||||||
" {blank}> ",
|
|
||||||
blank = " ".repeat(ks.len() + tbl.len() + 1)
|
|
||||||
);
|
|
||||||
*prompt = format!("skysh@{ks}:{tbl}> ", ks = ks, tbl = tbl);
|
|
||||||
}
|
|
||||||
count => fatal!(
|
|
||||||
"The server returned {} IDs while checking entity state",
|
|
||||||
count
|
|
||||||
),
|
|
||||||
},
|
|
||||||
_ => fatal!("The server returned the wrong data type for entity state check"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_float(float: f32, idx: Option<usize>) {
|
|
||||||
if let Some(idx) = idx {
|
|
||||||
println!("({idx}) {float}")
|
|
||||||
} else {
|
|
||||||
println!("{float}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_element(el: Element) {
|
|
||||||
match el {
|
|
||||||
Element::String(st) => write_str!(st),
|
|
||||||
Element::Binstr(st) => write_binstr!(st),
|
|
||||||
Element::Array(Array::Bin(brr)) => print_bin_array(brr),
|
|
||||||
Element::Array(Array::Str(srr)) => print_str_array(srr),
|
|
||||||
Element::RespCode(r) => print_rcode(r, None),
|
|
||||||
Element::UnsignedInt(int) => write_int!(int),
|
|
||||||
Element::Array(Array::Flat(frr)) => write_flat_array(frr),
|
|
||||||
Element::Array(Array::Recursive(a)) => print_array(a),
|
|
||||||
Element::Array(Array::NonNullBin(nbrr)) => print_array_nonnull_bin(nbrr),
|
|
||||||
Element::Array(Array::NonNullStr(nsrr)) => print_array_nonnull_str(nsrr),
|
|
||||||
Element::Float(float) => print_float(float, None),
|
|
||||||
_ => eskysh!("The server possibly sent a newer data type that we can't parse"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_rcode(rcode: RespCode, idx: Option<usize>) {
|
|
||||||
match rcode {
|
|
||||||
RespCode::Okay => write_okay!(),
|
|
||||||
RespCode::ActionError => write_err!(idx, "Action Error"),
|
|
||||||
RespCode::ErrorString(st) => write_err!(idx, st),
|
|
||||||
RespCode::OtherError => write_err!(idx, "Other Error"),
|
|
||||||
RespCode::NotFound => write_err!(idx, "Not Found"),
|
|
||||||
RespCode::OverwriteError => write_err!(idx, "Overwrite Error"),
|
|
||||||
RespCode::PacketError => write_err!(idx, "Packet Error"),
|
|
||||||
RespCode::ServerError => write_err!(idx, "Server Error"),
|
|
||||||
RespCode::UnknownDataType => write_err!(idx, "Unknown data type"),
|
|
||||||
RespCode::EncodingError => write_err!(idx, "Encoding error"),
|
|
||||||
RespCode::AuthBadCredentials => write_err!(idx, "auth bad credentials"),
|
|
||||||
RespCode::AuthPermissionError => write_err!(idx, "auth permission error"),
|
|
||||||
_ => write_err!(idx, "Unknown error"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_bin_array(bin_array: Vec<Option<Vec<u8>>>) {
|
|
||||||
bin_array.into_iter().enumerate().for_each(|(idx, elem)| {
|
|
||||||
let idx = idx + 1;
|
|
||||||
match elem {
|
|
||||||
Some(ele) => {
|
|
||||||
write_binstr!(idx, ele);
|
|
||||||
}
|
|
||||||
None => print_rcode(RespCode::NotFound, Some(idx)),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_str_array(str_array: Vec<Option<String>>) {
|
|
||||||
str_array.into_iter().enumerate().for_each(|(idx, elem)| {
|
|
||||||
let idx = idx + 1;
|
|
||||||
match elem {
|
|
||||||
Some(ele) => {
|
|
||||||
write_str!(idx, ele);
|
|
||||||
}
|
|
||||||
None => print_rcode(RespCode::NotFound, Some(idx)),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_array_nonnull_str(str_array: Vec<String>) {
|
|
||||||
str_array.into_iter().enumerate().for_each(|(idx, elem)| {
|
|
||||||
let idx = idx + 1;
|
|
||||||
write_str!(idx, elem)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_array_nonnull_bin(str_array: Vec<Vec<u8>>) {
|
|
||||||
str_array.into_iter().enumerate().for_each(|(idx, elem)| {
|
|
||||||
let idx = idx + 1;
|
|
||||||
write_binstr!(idx, elem)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_flat_array(flat_array: Vec<FlatElement>) {
|
|
||||||
for (idx, item) in flat_array.into_iter().enumerate() {
|
|
||||||
let idx = idx + 1;
|
|
||||||
match item {
|
|
||||||
FlatElement::String(st) => write_str!(idx, st),
|
|
||||||
FlatElement::Binstr(st) => {
|
|
||||||
write_binstr!(idx, st)
|
|
||||||
}
|
|
||||||
FlatElement::RespCode(rc) => print_rcode(rc, Some(idx)),
|
|
||||||
FlatElement::UnsignedInt(int) => write_int!(int, idx),
|
|
||||||
_ => eskysh!("Element typed cannot yet be parsed"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn print_array(array: Vec<Element>) {
|
|
||||||
for (idx, item) in array.into_iter().enumerate() {
|
|
||||||
let idx = idx + 1;
|
|
||||||
match item {
|
|
||||||
Element::String(st) => write_str!(idx, st),
|
|
||||||
Element::RespCode(rc) => print_rcode(rc, Some(idx)),
|
|
||||||
Element::UnsignedInt(int) => write_int!(idx, int),
|
|
||||||
Element::Array(Array::Bin(brr)) => print_bin_array(brr),
|
|
||||||
Element::Array(Array::Str(srr)) => print_str_array(srr),
|
|
||||||
_ => eskysh!("Nested arrays cannot be printed just yet"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct BinaryData(Vec<u8>);
|
|
||||||
|
|
||||||
impl fmt::Display for BinaryData {
|
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
|
|
||||||
write!(f, "b\"")?;
|
|
||||||
for b in self.0.iter() {
|
|
||||||
let b = *b;
|
|
||||||
// See this: https://doc.rust-lang.org/reference/tokens.html#byte-escapes
|
|
||||||
// this idea was borrowed from the Bytes crate
|
|
||||||
#[allow(clippy::manual_range_contains)]
|
|
||||||
if b == b'\n' {
|
|
||||||
write!(f, "\\n")?;
|
|
||||||
} else if b == b'\r' {
|
|
||||||
write!(f, "\\r")?;
|
|
||||||
} else if b == b'\t' {
|
|
||||||
write!(f, "\\t")?;
|
|
||||||
} else if b == b'\\' || b == b'"' {
|
|
||||||
write!(f, "\\{}", b as char)?;
|
|
||||||
} else if b == b'\0' {
|
|
||||||
write!(f, "\\0")?;
|
|
||||||
// ASCII printable
|
|
||||||
} else if b >= 0x20 && b < 0x7f {
|
|
||||||
write!(f, "{}", b as char)?;
|
|
||||||
} else {
|
|
||||||
write!(f, "\\x{:02x}", b)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
write!(f, "\"")?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,196 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Sun Oct 10 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use crate::tokenizer::{get_query, TokenizerError};
|
|
||||||
|
|
||||||
fn query_from(input: &[u8]) -> Result<Vec<String>, TokenizerError> {
|
|
||||||
get_query(input)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_basic_tokenization() {
|
|
||||||
let input = "set x 100".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_single_quote_tokens() {
|
|
||||||
let input = "set 'x with a whitespace' 100".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec![
|
|
||||||
"set".to_owned(),
|
|
||||||
"x with a whitespace".to_owned(),
|
|
||||||
"100".to_owned()
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_double_quote_tokens() {
|
|
||||||
let input = r#"set "x with a whitespace" 100"#.as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec![
|
|
||||||
"set".to_owned(),
|
|
||||||
"x with a whitespace".to_owned(),
|
|
||||||
"100".to_owned()
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_single_and_double_quote_tokens() {
|
|
||||||
let input = r#"set "x with a whitespace" 'y with a whitespace'"#.as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec![
|
|
||||||
"set".to_owned(),
|
|
||||||
"x with a whitespace".to_owned(),
|
|
||||||
"y with a whitespace".to_owned()
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_multiple_single_quote_tokens() {
|
|
||||||
let input = r#"'set' 'x with a whitespace' 'y with a whitespace'"#.as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec![
|
|
||||||
"set".to_owned(),
|
|
||||||
"x with a whitespace".to_owned(),
|
|
||||||
"y with a whitespace".to_owned()
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_multiple_double_quote_tokens() {
|
|
||||||
let input = r#""set" "x with a whitespace" "y with a whitespace""#.as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec![
|
|
||||||
"set".to_owned(),
|
|
||||||
"x with a whitespace".to_owned(),
|
|
||||||
"y with a whitespace".to_owned()
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_missing_single_quote() {
|
|
||||||
let input = r#"'get' 'x with a whitespace"#.as_bytes();
|
|
||||||
let ret = format!("{}", query_from(input).unwrap_err());
|
|
||||||
assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_missing_double_quote() {
|
|
||||||
let input = r#"'get' "x with a whitespace"#.as_bytes();
|
|
||||||
let ret = format!("{}", query_from(input).unwrap_err());
|
|
||||||
assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_extra_whitespace() {
|
|
||||||
let input = "set x '100'".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_singly_quoted() {
|
|
||||||
let input = "set tables' wth".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap_err();
|
|
||||||
assert_eq!(ret, TokenizerError::ExpectedWhitespace("tables".to_owned()));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_text_after_quote_nospace() {
|
|
||||||
let input = "get 'rust'ferris".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap_err();
|
|
||||||
assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust'".to_owned()));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_text_after_double_quote_nospace() {
|
|
||||||
let input = r#"get "rust"ferris"#.as_bytes();
|
|
||||||
let ret = query_from(input).unwrap_err();
|
|
||||||
assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust\"".to_owned()));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_inline_comment() {
|
|
||||||
let input = "set x 100 # sets x to 100".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
ret,
|
|
||||||
vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_full_comment() {
|
|
||||||
let input = "# what is going on?".as_bytes();
|
|
||||||
let ret = query_from(input).unwrap();
|
|
||||||
assert!(ret.is_empty());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_ignore_comment() {
|
|
||||||
let input = "set x \"# ooh la la\"".as_bytes();
|
|
||||||
assert_eq!(
|
|
||||||
query_from(input).unwrap(),
|
|
||||||
vec!["set".to_owned(), "x".to_owned(), "# ooh la la".to_owned()]
|
|
||||||
);
|
|
||||||
let input = "set x \"#\"".as_bytes();
|
|
||||||
assert_eq!(
|
|
||||||
query_from(input).unwrap(),
|
|
||||||
vec!["set".to_owned(), "x".to_owned(), "#".to_owned()]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_blueql_query() {
|
|
||||||
let input = b"create model mymodel(string, binary)";
|
|
||||||
assert_eq!(
|
|
||||||
query_from(input).unwrap(),
|
|
||||||
vec!["create model mymodel(string, binary)"]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@ -1,226 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Sat Oct 09 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! This module provides a simple way to avoid "the funk" with "funky input queries". It simply
|
|
||||||
//! tokenizes char-by-char analyzing quotes et al as required
|
|
||||||
//!
|
|
||||||
|
|
||||||
use {
|
|
||||||
core::fmt,
|
|
||||||
skytable::{types::RawString, Query},
|
|
||||||
std::collections::HashSet,
|
|
||||||
};
|
|
||||||
|
|
||||||
lazy_static::lazy_static! {
|
|
||||||
static ref BLUEQL_KW: HashSet<&'static [u8]> = {
|
|
||||||
let mut hs = HashSet::new();
|
|
||||||
hs.insert("create".as_bytes());
|
|
||||||
hs.insert(b"inspect");
|
|
||||||
hs.insert(b"drop");
|
|
||||||
hs.insert(b"use");
|
|
||||||
hs
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq)]
|
|
||||||
pub enum TokenizerError {
|
|
||||||
QuoteMismatch(String),
|
|
||||||
BacktickMismatch(String),
|
|
||||||
ExpectedWhitespace(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Display for TokenizerError {
|
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
match self {
|
|
||||||
Self::QuoteMismatch(expr) => write!(f, "mismatched quotes near end of: `{}`", expr),
|
|
||||||
Self::ExpectedWhitespace(expr) => {
|
|
||||||
write!(f, "expected whitespace near end of: `{}`", expr)
|
|
||||||
}
|
|
||||||
Self::BacktickMismatch(expr) => {
|
|
||||||
write!(f, "mismatched backticks near end of: `{}`", expr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait SequentialQuery {
|
|
||||||
fn push(&mut self, input: &[u8]);
|
|
||||||
fn new() -> Self;
|
|
||||||
fn is_empty(&self) -> bool;
|
|
||||||
}
|
|
||||||
|
|
||||||
// #[cfg(test)]
|
|
||||||
impl SequentialQuery for Vec<String> {
|
|
||||||
fn push(&mut self, input: &[u8]) {
|
|
||||||
Vec::push(self, String::from_utf8_lossy(input).to_string())
|
|
||||||
}
|
|
||||||
fn is_empty(&self) -> bool {
|
|
||||||
Vec::len(self) == 0
|
|
||||||
}
|
|
||||||
fn new() -> Self {
|
|
||||||
Vec::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SequentialQuery for Query {
|
|
||||||
fn push(&mut self, input: &[u8]) {
|
|
||||||
Query::push(self, RawString::from(input.to_owned()))
|
|
||||||
}
|
|
||||||
fn is_empty(&self) -> bool {
|
|
||||||
Query::len(self) == 0
|
|
||||||
}
|
|
||||||
fn new() -> Self {
|
|
||||||
Query::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME(@ohsayan): Fix this entire impl. At this point, it's almost like legacy code
|
|
||||||
pub fn get_query<T: SequentialQuery>(inp: &[u8]) -> Result<T, TokenizerError> {
|
|
||||||
assert!(!inp.is_empty(), "Input is empty");
|
|
||||||
let mut query = T::new();
|
|
||||||
let mut it = inp.iter().peekable();
|
|
||||||
macro_rules! pos {
|
|
||||||
() => {
|
|
||||||
inp.len() - it.len()
|
|
||||||
};
|
|
||||||
}
|
|
||||||
macro_rules! expect_whitespace {
|
|
||||||
($start:expr) => {
|
|
||||||
match it.peek() {
|
|
||||||
Some(b) => match **b {
|
|
||||||
b' ' => {}
|
|
||||||
_ => {
|
|
||||||
return Err(TokenizerError::ExpectedWhitespace(
|
|
||||||
String::from_utf8_lossy(&inp[$start..pos!()]).to_string(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
},
|
|
||||||
None => {}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
// skip useless starting whitespace
|
|
||||||
while let Some(b' ') = it.next() {}
|
|
||||||
let end_of_first = match it.position(|x| *x == b' ') {
|
|
||||||
Some(e) => e + 1,
|
|
||||||
None if it.len() == 0 => inp.len(),
|
|
||||||
None => {
|
|
||||||
return Err(TokenizerError::ExpectedWhitespace(
|
|
||||||
String::from_utf8_lossy(inp).to_string(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if BLUEQL_KW.contains(inp[..end_of_first].to_ascii_lowercase().as_slice()) {
|
|
||||||
query.push(inp);
|
|
||||||
return Ok(query);
|
|
||||||
} else {
|
|
||||||
it = inp.iter().peekable();
|
|
||||||
}
|
|
||||||
'outer: while let Some(tok) = it.next() {
|
|
||||||
match tok {
|
|
||||||
b'\'' => {
|
|
||||||
// hmm, quotes; let's see where it ends
|
|
||||||
let pos = pos!();
|
|
||||||
let qidx = it.position(|x| *x == b'\'');
|
|
||||||
match qidx {
|
|
||||||
Some(idx) => query.push(&inp[pos..idx + pos]),
|
|
||||||
None => {
|
|
||||||
let end = pos!();
|
|
||||||
return Err(TokenizerError::QuoteMismatch(
|
|
||||||
String::from_utf8_lossy(&inp[pos..end]).to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect_whitespace!(pos);
|
|
||||||
}
|
|
||||||
b'"' => {
|
|
||||||
// hmm, quotes; let's see where it ends
|
|
||||||
let pos = pos!();
|
|
||||||
let qidx = it.position(|x| *x == b'"');
|
|
||||||
match qidx {
|
|
||||||
Some(idx) => query.push(&inp[pos..idx + pos]),
|
|
||||||
None => {
|
|
||||||
let end = pos!();
|
|
||||||
return Err(TokenizerError::QuoteMismatch(
|
|
||||||
String::from_utf8_lossy(&inp[pos..end]).to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect_whitespace!(pos);
|
|
||||||
}
|
|
||||||
b'`' => {
|
|
||||||
// hmm, backtick? let's look for the end
|
|
||||||
let pos = pos!();
|
|
||||||
let qidx = it.position(|x| *x == b'`');
|
|
||||||
match qidx {
|
|
||||||
Some(idx) => query.push(&inp[pos..idx + pos]),
|
|
||||||
None => {
|
|
||||||
let end = pos!();
|
|
||||||
return Err(TokenizerError::BacktickMismatch(
|
|
||||||
String::from_utf8_lossy(&inp[pos..end]).to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect_whitespace!(pos);
|
|
||||||
}
|
|
||||||
b' ' => {
|
|
||||||
// this just prevents control from being handed to the wildcard
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
b'#' => {
|
|
||||||
// so this is an inline comment; skip until newline
|
|
||||||
let _ = it.position(|x| *x == b'\n');
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
let start = pos!() - 1;
|
|
||||||
let mut end = start;
|
|
||||||
// alpha? cool, go on
|
|
||||||
'inner: while let Some(tok) = it.peek() {
|
|
||||||
match **tok {
|
|
||||||
b' ' => {
|
|
||||||
it.next();
|
|
||||||
break 'inner;
|
|
||||||
}
|
|
||||||
b'\'' | b'"' => {
|
|
||||||
return Err(TokenizerError::ExpectedWhitespace(
|
|
||||||
String::from_utf8_lossy(&inp[start..pos!()]).to_string(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
b'#' => continue 'outer,
|
|
||||||
_ => {
|
|
||||||
end += 1;
|
|
||||||
it.next();
|
|
||||||
continue 'inner;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end += 1;
|
|
||||||
query.push(&inp[start..end]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(query)
|
|
||||||
}
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
# This is a 'bad' configuration file since it contains an invalid port
|
|
||||||
[server]
|
|
||||||
port = 20033002
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
# This is a bad toml file since it contains an invalid value of `every` in BGSAVE
|
|
||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
every = 0.5 # Not possible!
|
|
||||||
enabled = true
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
enabled = true
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
every = 600
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
# this is the default configuration file for Docker images. Modify it as required
|
|
||||||
[server]
|
|
||||||
host = "0.0.0.0"
|
|
||||||
port = 2003
|
|
||||||
noart = true
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
# This makes use of an IPv6 address
|
|
||||||
[server]
|
|
||||||
host = "::1"
|
|
||||||
port = 2003
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = '127.0.0.1' # i.e localhost
|
|
||||||
port = 2003 # The port to bind to
|
|
||||||
noart = true # No terminal artwork
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
# This is a 'good' configuration file since it contains a valid port
|
|
||||||
# and appropriate keys
|
|
||||||
[server]
|
|
||||||
host = '127.0.0.1'
|
|
||||||
port = 2003 # Set the server's port to 2003
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1" # The IP address to which you want sdb to bind to
|
|
||||||
port = 2003 # The port to which you want sdb to bind to
|
|
||||||
# Set `noart` to true if you want to disable terminal artwork
|
|
||||||
noart = false
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run
|
|
||||||
# after every 2 minutes
|
|
||||||
enabled = true
|
|
||||||
every = 120
|
|
||||||
|
|
||||||
[snapshot]
|
|
||||||
# Create a snapshot every hour (1 hour = 60 minutes = 60 * 60 seconds = 3600 seconds)
|
|
||||||
every = 3600
|
|
||||||
# How many of the snapshots to keep
|
|
||||||
atmost = 4 # keep the four most recent snapshots
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
noart = false
|
|
||||||
|
|
||||||
[ssl]
|
|
||||||
key = "/path/to/keyfile.pem"
|
|
||||||
chain = "/path/to/chain.pem"
|
|
||||||
port = 2004
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
enabled = true
|
|
||||||
every = 120
|
|
||||||
|
|
||||||
[snapshot]
|
|
||||||
every = 3600
|
|
||||||
atmost = 4
|
|
||||||
@ -1,40 +0,0 @@
|
|||||||
# This is a complete sdb configuration template which is always kept updated
|
|
||||||
# to include all the configuration options. I encourage you to always use this
|
|
||||||
# when you use a configuration file
|
|
||||||
# Instead of deleting entire sections from this file, comment them out, so that you
|
|
||||||
# now what you've kept enabled and what you've kept disabled. This helps avoid
|
|
||||||
# configuration problems during production
|
|
||||||
|
|
||||||
# This is a *REQUIRED* key
|
|
||||||
[server]
|
|
||||||
host = "127.0.0.1" # The IP address to which you want sdb to bind to
|
|
||||||
port = 2003 # The port to which you want sdb to bind to
|
|
||||||
noart = false # Set `noart` to true if you want to disable terminal artwork
|
|
||||||
maxcon = 50000 # set the maximum number of clients that the server can accept
|
|
||||||
mode = "dev" # Set this to `prod` when you're running in production and `dev` when in development
|
|
||||||
|
|
||||||
# This is an optional key
|
|
||||||
[auth]
|
|
||||||
# the origin key to be used to claim the root account
|
|
||||||
origin_key = "4527387f92a381cbe804593f33991d327d456a97"
|
|
||||||
|
|
||||||
# This key is *OPTIONAL*
|
|
||||||
[bgsave]
|
|
||||||
# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run
|
|
||||||
# after every 2 minutes
|
|
||||||
enabled = true
|
|
||||||
every = 120
|
|
||||||
|
|
||||||
# This key is *OPTIONAL*
|
|
||||||
[snapshot]
|
|
||||||
every = 3600 # Make a snapshot after every 1 hour (60min * 60sec= 3600secs)
|
|
||||||
atmost = 4 # Keep the 4 most recent snapshots
|
|
||||||
failsafe = true # stops accepting writes if snapshotting fails
|
|
||||||
|
|
||||||
# This key is *OPTIONAL*, used for TLS/SSL config
|
|
||||||
[ssl]
|
|
||||||
key = "/path/to/keyfile.pem"
|
|
||||||
chain = "/path/to/chain.pem"
|
|
||||||
port = 2004
|
|
||||||
only = true # optional to enable SSL-only requests
|
|
||||||
passin = "/path/to/cert/passphrase.txt" # optional to programmatically verify the TLS cert
|
|
||||||
@ -0,0 +1,22 @@
|
|||||||
|
system:
|
||||||
|
mode: prod
|
||||||
|
rs_window: 600
|
||||||
|
|
||||||
|
auth:
|
||||||
|
plugin: pwd
|
||||||
|
# replace with your root password of choice
|
||||||
|
root_pass: password
|
||||||
|
|
||||||
|
endpoints:
|
||||||
|
secure:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 2004
|
||||||
|
# replace `cert` with the path to your self-signed certificate
|
||||||
|
cert: cert.pem
|
||||||
|
# replace `private_key` with the path to your private key
|
||||||
|
private_key: private.key
|
||||||
|
# replace `passphrase.txt` with the path to your private key passphrase
|
||||||
|
pkey_passphrase: passphrase.txt
|
||||||
|
insecure:
|
||||||
|
host: 127.0.0.1
|
||||||
|
port: 2003
|
||||||
@ -1,7 +0,0 @@
|
|||||||
[server]
|
|
||||||
host = "127.0.0.1"
|
|
||||||
port = 2003
|
|
||||||
|
|
||||||
[bgsave]
|
|
||||||
enabled = true
|
|
||||||
every = 600 # Every 10 minutes
|
|
||||||
@ -1,5 +1,5 @@
|
|||||||
Skytable is a free and open-source NoSQL database that aims
|
Skytable is a free and open-source NoSQL database that aims
|
||||||
to provide flexibility in data modeling at scale.
|
to provide flexibility in data modeling at scale.
|
||||||
The `skytable` package contains the database server (`skyd`),
|
The `skytable` package contains the database server (`skyd`),
|
||||||
an interactive command-line client (`skysh`), a benchmarking
|
an interactive command-line client (`skysh`) and a benchmarking
|
||||||
tool (`sky-bench`) and a migration tool (`sky-migrate`).
|
tool (`sky-bench`).
|
||||||
@ -1,123 +0,0 @@
|
|||||||
name: Skytable Server
|
|
||||||
version: 0.8.0
|
|
||||||
author: Sayan N. <ohsayan@outlook.com>
|
|
||||||
about: The Skytable Database server
|
|
||||||
args:
|
|
||||||
- config:
|
|
||||||
short: c
|
|
||||||
required: false
|
|
||||||
long: withconfig
|
|
||||||
value_name: cfgfile
|
|
||||||
help: Sets a configuration file to start skyd
|
|
||||||
takes_value: true
|
|
||||||
- restore:
|
|
||||||
short: r
|
|
||||||
required: false
|
|
||||||
long: restore
|
|
||||||
value_name: backupdir
|
|
||||||
help: Restores data from a previous snapshot made in the provided directory
|
|
||||||
takes_value: true
|
|
||||||
- host:
|
|
||||||
short: h
|
|
||||||
required: false
|
|
||||||
long: host
|
|
||||||
value_name: host
|
|
||||||
help: Sets the host to which the server will bind
|
|
||||||
takes_value: true
|
|
||||||
- port:
|
|
||||||
short: p
|
|
||||||
required: false
|
|
||||||
long: port
|
|
||||||
value_name: port
|
|
||||||
help: Sets the port to which the server will bind
|
|
||||||
takes_value: true
|
|
||||||
- noart:
|
|
||||||
required: false
|
|
||||||
long: noart
|
|
||||||
help: Disables terminal artwork
|
|
||||||
takes_value: false
|
|
||||||
- nosave:
|
|
||||||
required: false
|
|
||||||
long: nosave
|
|
||||||
help: Disables automated background saving
|
|
||||||
takes_value: false
|
|
||||||
- saveduration:
|
|
||||||
required: false
|
|
||||||
long: saveduration
|
|
||||||
value_name: duration
|
|
||||||
short: S
|
|
||||||
takes_value: true
|
|
||||||
help: Set the BGSAVE duration
|
|
||||||
- snapevery:
|
|
||||||
required: false
|
|
||||||
long: snapevery
|
|
||||||
value_name: duration
|
|
||||||
help: Set the periodic snapshot duration
|
|
||||||
takes_value: true
|
|
||||||
- snapkeep:
|
|
||||||
required: false
|
|
||||||
long: snapkeep
|
|
||||||
value_name: count
|
|
||||||
help: Sets the number of most recent snapshots to keep
|
|
||||||
takes_value: true
|
|
||||||
- sslkey:
|
|
||||||
required: false
|
|
||||||
long: sslkey
|
|
||||||
short: k
|
|
||||||
value_name: key
|
|
||||||
help: Sets the PEM key file to use for SSL/TLS
|
|
||||||
takes_value: true
|
|
||||||
- sslchain:
|
|
||||||
required: false
|
|
||||||
long: sslchain
|
|
||||||
short: z
|
|
||||||
value_name: chain
|
|
||||||
help: Sets the PEM chain file to use for SSL/TLS
|
|
||||||
takes_value: true
|
|
||||||
- sslonly:
|
|
||||||
required: false
|
|
||||||
long: sslonly
|
|
||||||
takes_value: false
|
|
||||||
help: Tells the server to only accept SSL connections and disables the non-SSL port
|
|
||||||
- sslport:
|
|
||||||
required: false
|
|
||||||
long: sslport
|
|
||||||
takes_value: true
|
|
||||||
value_name: sslport
|
|
||||||
help: Set a custom SSL port to bind to
|
|
||||||
- tlspassin:
|
|
||||||
required: false
|
|
||||||
long: tlspassin
|
|
||||||
takes_value: true
|
|
||||||
value_name: tlspassin
|
|
||||||
help: Path to the file containing the passphrase for the TLS certificate
|
|
||||||
- stopwriteonfail:
|
|
||||||
required: false
|
|
||||||
long: stop-write-on-fail
|
|
||||||
takes_value: true
|
|
||||||
help: Stop accepting writes if any persistence method except BGSAVE fails (defaults to true)
|
|
||||||
- maxcon:
|
|
||||||
required: false
|
|
||||||
long: maxcon
|
|
||||||
takes_value: true
|
|
||||||
help: Set the maximum number of connections
|
|
||||||
value_name: maxcon
|
|
||||||
- mode:
|
|
||||||
required: false
|
|
||||||
long: mode
|
|
||||||
takes_value: true
|
|
||||||
short: m
|
|
||||||
help: Sets the deployment type
|
|
||||||
value_name: mode
|
|
||||||
- authkey:
|
|
||||||
required: false
|
|
||||||
long: auth-origin-key
|
|
||||||
takes_value: true
|
|
||||||
help: Set the authentication origin key
|
|
||||||
value_name: origin_key
|
|
||||||
- protover:
|
|
||||||
required: false
|
|
||||||
long: protover
|
|
||||||
takes_value: true
|
|
||||||
help: Set the protocol version
|
|
||||||
value_name: protover
|
|
||||||
@ -1,245 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Sat Aug 13 2022
|
|
||||||
*
|
|
||||||
* This file is a part of S{
|
|
||||||
let ref this = loopmon;
|
|
||||||
this.current
|
|
||||||
}le (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
super::{
|
|
||||||
report::{AggregateReport, SingleReport},
|
|
||||||
validation, vec_with_cap, BenchmarkConfig, LoopMonitor,
|
|
||||||
},
|
|
||||||
crate::error::BResult,
|
|
||||||
devtimer::SimpleTimer,
|
|
||||||
libstress::Workpool,
|
|
||||||
skytable::{types::RawString, Connection, Element, Query, RespCode},
|
|
||||||
std::{
|
|
||||||
io::{Read, Write},
|
|
||||||
net::{Shutdown, TcpStream},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Run a benchmark using the given pre-loop, in-loop and post-loop closures
|
|
||||||
fn run_bench_custom<Inp, Lp, Lv, Ex>(
|
|
||||||
bench_config: BenchmarkConfig,
|
|
||||||
packets: Vec<Box<[u8]>>,
|
|
||||||
on_init: Lv,
|
|
||||||
on_loop: Lp,
|
|
||||||
on_loop_exit: Ex,
|
|
||||||
loopmon: LoopMonitor,
|
|
||||||
reports: &mut AggregateReport,
|
|
||||||
) -> BResult<()>
|
|
||||||
where
|
|
||||||
Ex: Clone + Fn(&mut Inp) + Send + Sync + 'static,
|
|
||||||
Inp: Sync + 'static,
|
|
||||||
Lp: Clone + Fn(&mut Inp, Box<[u8]>) + Send + Sync + 'static,
|
|
||||||
Lv: Clone + Fn() -> Inp + Send + 'static + Sync,
|
|
||||||
{
|
|
||||||
// now do our runs
|
|
||||||
let mut loopmon = loopmon;
|
|
||||||
|
|
||||||
while loopmon.should_continue() {
|
|
||||||
// now create our connection pool
|
|
||||||
let pool = Workpool::new(
|
|
||||||
bench_config.server.connections(),
|
|
||||||
on_init.clone(),
|
|
||||||
on_loop.clone(),
|
|
||||||
on_loop_exit.clone(),
|
|
||||||
true,
|
|
||||||
Some(bench_config.query_count()),
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// get our local copy
|
|
||||||
let this_packets = packets.clone();
|
|
||||||
|
|
||||||
// run and time our operations
|
|
||||||
let mut dt = SimpleTimer::new();
|
|
||||||
dt.start();
|
|
||||||
pool.execute_and_finish_iter(this_packets);
|
|
||||||
dt.stop();
|
|
||||||
loopmon.incr_time(&dt);
|
|
||||||
|
|
||||||
// cleanup
|
|
||||||
loopmon.cleanup()?;
|
|
||||||
loopmon.step();
|
|
||||||
}
|
|
||||||
|
|
||||||
// save time
|
|
||||||
reports.push(SingleReport::new(
|
|
||||||
loopmon.name(),
|
|
||||||
loopmon.sum() as f64 / bench_config.runs() as f64,
|
|
||||||
));
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
/// Init connection and buffer
|
|
||||||
fn init_connection_and_buf(
|
|
||||||
host: &str,
|
|
||||||
port: u16,
|
|
||||||
start_command: Vec<u8>,
|
|
||||||
bufsize: usize,
|
|
||||||
) -> (TcpStream, Vec<u8>) {
|
|
||||||
let mut con = TcpStream::connect((host, port)).unwrap();
|
|
||||||
con.write_all(&start_command).unwrap();
|
|
||||||
let mut ret = [0u8; validation::RESPCODE_OKAY.len()];
|
|
||||||
con.read_exact(&mut ret).unwrap();
|
|
||||||
let readbuf = vec![0; bufsize];
|
|
||||||
(con, readbuf)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Benchmark SET
|
|
||||||
pub fn bench_set(
|
|
||||||
keys: &[Vec<u8>],
|
|
||||||
values: &[Vec<u8>],
|
|
||||||
connection: &mut Connection,
|
|
||||||
bench_config: &BenchmarkConfig,
|
|
||||||
create_table: &[u8],
|
|
||||||
reports: &mut AggregateReport,
|
|
||||||
) -> BResult<()> {
|
|
||||||
let bench_config = bench_config.clone();
|
|
||||||
let create_table = create_table.to_owned();
|
|
||||||
let loopmon = LoopMonitor::new_cleanup(
|
|
||||||
bench_config.runs(),
|
|
||||||
"set",
|
|
||||||
connection,
|
|
||||||
Query::from("FLUSHDB").arg("default.tmpbench"),
|
|
||||||
Element::RespCode(RespCode::Okay),
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
let mut packets = vec_with_cap(bench_config.query_count())?;
|
|
||||||
(0..bench_config.query_count()).for_each(|i| {
|
|
||||||
packets.push(
|
|
||||||
Query::from("SET")
|
|
||||||
.arg(RawString::from(keys[i].to_owned()))
|
|
||||||
.arg(RawString::from(values[i].to_owned()))
|
|
||||||
.into_raw_query()
|
|
||||||
.into_boxed_slice(),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
run_bench_custom(
|
|
||||||
bench_config.clone(),
|
|
||||||
packets,
|
|
||||||
move || {
|
|
||||||
init_connection_and_buf(
|
|
||||||
bench_config.server.host(),
|
|
||||||
bench_config.server.port(),
|
|
||||||
create_table.to_owned(),
|
|
||||||
validation::RESPCODE_OKAY.len(),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
|(con, buf), packet| {
|
|
||||||
con.write_all(&packet).unwrap();
|
|
||||||
con.read_exact(buf).unwrap();
|
|
||||||
assert_eq!(buf, validation::RESPCODE_OKAY);
|
|
||||||
},
|
|
||||||
|(con, _)| con.shutdown(Shutdown::Both).unwrap(),
|
|
||||||
loopmon,
|
|
||||||
reports,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Benchmark UPDATE
|
|
||||||
pub fn bench_update(
|
|
||||||
keys: &[Vec<u8>],
|
|
||||||
new_value: &[u8],
|
|
||||||
bench_config: &BenchmarkConfig,
|
|
||||||
create_table: &[u8],
|
|
||||||
reports: &mut AggregateReport,
|
|
||||||
) -> BResult<()> {
|
|
||||||
let bench_config = bench_config.clone();
|
|
||||||
let create_table = create_table.to_owned();
|
|
||||||
let loopmon = LoopMonitor::new(bench_config.runs(), "update");
|
|
||||||
let mut packets = vec_with_cap(bench_config.query_count())?;
|
|
||||||
(0..bench_config.query_count()).for_each(|i| {
|
|
||||||
packets.push(
|
|
||||||
Query::from("update")
|
|
||||||
.arg(RawString::from(keys[i].clone()))
|
|
||||||
.arg(RawString::from(new_value.to_owned()))
|
|
||||||
.into_raw_query()
|
|
||||||
.into_boxed_slice(),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
run_bench_custom(
|
|
||||||
bench_config.clone(),
|
|
||||||
packets,
|
|
||||||
move || {
|
|
||||||
init_connection_and_buf(
|
|
||||||
bench_config.server.host(),
|
|
||||||
bench_config.server.port(),
|
|
||||||
create_table.to_owned(),
|
|
||||||
validation::RESPCODE_OKAY.len(),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
|(con, buf), packet| {
|
|
||||||
con.write_all(&packet).unwrap();
|
|
||||||
con.read_exact(buf).unwrap();
|
|
||||||
assert_eq!(buf, validation::RESPCODE_OKAY);
|
|
||||||
},
|
|
||||||
|(con, _)| con.shutdown(Shutdown::Both).unwrap(),
|
|
||||||
loopmon,
|
|
||||||
reports,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Benchmark GET
|
|
||||||
pub fn bench_get(
|
|
||||||
keys: &[Vec<u8>],
|
|
||||||
bench_config: &BenchmarkConfig,
|
|
||||||
create_table: &[u8],
|
|
||||||
reports: &mut AggregateReport,
|
|
||||||
) -> BResult<()> {
|
|
||||||
let bench_config = bench_config.clone();
|
|
||||||
let create_table = create_table.to_owned();
|
|
||||||
let loopmon = LoopMonitor::new(bench_config.runs(), "get");
|
|
||||||
let mut packets = vec_with_cap(bench_config.query_count())?;
|
|
||||||
(0..bench_config.query_count()).for_each(|i| {
|
|
||||||
packets.push(
|
|
||||||
Query::from("get")
|
|
||||||
.arg(RawString::from(keys[i].clone()))
|
|
||||||
.into_raw_query()
|
|
||||||
.into_boxed_slice(),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
run_bench_custom(
|
|
||||||
bench_config.clone(),
|
|
||||||
packets,
|
|
||||||
move || {
|
|
||||||
init_connection_and_buf(
|
|
||||||
bench_config.server.host(),
|
|
||||||
bench_config.server.port(),
|
|
||||||
create_table.to_owned(),
|
|
||||||
validation::calculate_response_size(bench_config.kvsize()),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
|(con, buf), packet| {
|
|
||||||
con.write_all(&packet).unwrap();
|
|
||||||
con.read_exact(buf).unwrap();
|
|
||||||
},
|
|
||||||
|(con, _)| con.shutdown(Shutdown::Both).unwrap(),
|
|
||||||
loopmon,
|
|
||||||
reports,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@ -1,276 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Tue Aug 09 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
self::report::AggregateReport,
|
|
||||||
crate::{
|
|
||||||
config,
|
|
||||||
config::{BenchmarkConfig, ServerConfig},
|
|
||||||
error::{BResult, Error},
|
|
||||||
util,
|
|
||||||
},
|
|
||||||
clap::ArgMatches,
|
|
||||||
devtimer::SimpleTimer,
|
|
||||||
libstress::utils::{generate_random_byte_vector, ran_bytes},
|
|
||||||
skytable::{Connection, Element, Query, RespCode},
|
|
||||||
};
|
|
||||||
|
|
||||||
mod benches;
|
|
||||||
mod report;
|
|
||||||
mod validation;
|
|
||||||
|
|
||||||
macro_rules! binfo {
|
|
||||||
($($arg:tt)+) => {
|
|
||||||
if $crate::config::should_output_messages() {
|
|
||||||
::log::info!($($arg)+)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The loop monitor can be used for maintaining a loop for a given benchmark
|
|
||||||
struct LoopMonitor<'a> {
|
|
||||||
/// cleanup instructions
|
|
||||||
inner: Option<CleanupInner<'a>>,
|
|
||||||
/// maximum iterations
|
|
||||||
max: usize,
|
|
||||||
/// current iteration
|
|
||||||
current: usize,
|
|
||||||
/// total time
|
|
||||||
time: u128,
|
|
||||||
/// name of test
|
|
||||||
name: &'static str,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> LoopMonitor<'a> {
|
|
||||||
/// Create a benchmark loop monitor that doesn't need any cleanup
|
|
||||||
pub fn new(max: usize, name: &'static str) -> Self {
|
|
||||||
Self {
|
|
||||||
inner: None,
|
|
||||||
max,
|
|
||||||
current: 0,
|
|
||||||
time: 0,
|
|
||||||
name,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Create a new benchmark loop monitor that uses the given cleanup instructions:
|
|
||||||
/// - `max`: Total iterations
|
|
||||||
/// - `name`: Name of benchmark
|
|
||||||
/// - `connection`: A connection to use for cleanup instructions
|
|
||||||
/// - `query`: Query to run for cleanup
|
|
||||||
/// - `response`: Response expected when cleaned up
|
|
||||||
/// - `skip_on_last`: Skip running the cleanup instructions on the last loop
|
|
||||||
pub fn new_cleanup(
|
|
||||||
max: usize,
|
|
||||||
name: &'static str,
|
|
||||||
connection: &'a mut Connection,
|
|
||||||
query: Query,
|
|
||||||
response: Element,
|
|
||||||
skip_on_last: bool,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
inner: Some(CleanupInner::new(query, response, connection, skip_on_last)),
|
|
||||||
max,
|
|
||||||
current: 0,
|
|
||||||
time: 0,
|
|
||||||
name,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Run cleanup
|
|
||||||
fn cleanup(&mut self) -> BResult<()> {
|
|
||||||
let last_iter = self.is_last_iter();
|
|
||||||
if let Some(ref mut cleanup) = self.inner {
|
|
||||||
let should_run_cleanup = !(last_iter && cleanup.skip_on_last);
|
|
||||||
if should_run_cleanup {
|
|
||||||
return cleanup.cleanup(self.name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
/// Check if this is the last iteration
|
|
||||||
fn is_last_iter(&self) -> bool {
|
|
||||||
(self.max - 1) == self.current
|
|
||||||
}
|
|
||||||
/// Step the counter ahead
|
|
||||||
fn step(&mut self) {
|
|
||||||
self.current += 1;
|
|
||||||
}
|
|
||||||
/// Determine if we should continue executing
|
|
||||||
fn should_continue(&self) -> bool {
|
|
||||||
self.current < self.max
|
|
||||||
}
|
|
||||||
/// Append a new time to the sum
|
|
||||||
fn incr_time(&mut self, dt: &SimpleTimer) {
|
|
||||||
self.time += dt.time_in_nanos().unwrap();
|
|
||||||
}
|
|
||||||
/// Return the sum
|
|
||||||
fn sum(&self) -> u128 {
|
|
||||||
self.time
|
|
||||||
}
|
|
||||||
/// Return the name of the benchmark
|
|
||||||
fn name(&self) -> &'static str {
|
|
||||||
self.name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Cleanup instructions
|
|
||||||
struct CleanupInner<'a> {
|
|
||||||
/// the connection to use for cleanup processes
|
|
||||||
connection: &'a mut Connection,
|
|
||||||
/// the query to be run
|
|
||||||
query: Query,
|
|
||||||
/// the response to expect
|
|
||||||
response: Element,
|
|
||||||
/// whether we should skip on the last loop
|
|
||||||
skip_on_last: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a> CleanupInner<'a> {
|
|
||||||
/// Init cleanup instructions
|
|
||||||
fn new(q: Query, r: Element, connection: &'a mut Connection, skip_on_last: bool) -> Self {
|
|
||||||
Self {
|
|
||||||
query: q,
|
|
||||||
response: r,
|
|
||||||
connection,
|
|
||||||
skip_on_last,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Run cleanup
|
|
||||||
fn cleanup(&mut self, name: &'static str) -> BResult<()> {
|
|
||||||
let r: Element = self.connection.run_query(&self.query)?;
|
|
||||||
if r.ne(&self.response) {
|
|
||||||
Err(Error::Runtime(format!(
|
|
||||||
"Failed to run cleanup for benchmark `{}`",
|
|
||||||
name
|
|
||||||
)))
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
/// Returns a vec with the given cap, ensuring that we don't overflow memory
|
|
||||||
fn vec_with_cap<T>(cap: usize) -> BResult<Vec<T>> {
|
|
||||||
let mut v = Vec::new();
|
|
||||||
v.try_reserve_exact(cap)?;
|
|
||||||
Ok(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Run the actual benchmarks
|
|
||||||
pub fn run_bench(servercfg: &ServerConfig, matches: ArgMatches) -> BResult<()> {
|
|
||||||
// init bench config
|
|
||||||
let bench_config = BenchmarkConfig::new(servercfg, matches)?;
|
|
||||||
// check if we have enough combinations for the given query count and key size
|
|
||||||
if !util::has_enough_ncr(bench_config.kvsize(), bench_config.query_count()) {
|
|
||||||
return Err(Error::Runtime(
|
|
||||||
"too low sample space for given query count. use larger kvsize".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
// run sanity test; this will also set up the temporary table for benchmarking
|
|
||||||
binfo!("Running sanity test ...");
|
|
||||||
util::run_sanity_test(&bench_config.server)?;
|
|
||||||
|
|
||||||
// pool pre-exec setup
|
|
||||||
let servercfg = servercfg.clone();
|
|
||||||
let switch_table = Query::from("use default.tmpbench").into_raw_query();
|
|
||||||
|
|
||||||
// init pool config; side_connection is for cleanups
|
|
||||||
let mut misc_connection = Connection::new(servercfg.host(), servercfg.port())?;
|
|
||||||
|
|
||||||
// init timer and reports
|
|
||||||
let mut reports = AggregateReport::new(bench_config.query_count());
|
|
||||||
|
|
||||||
// init test data
|
|
||||||
binfo!("Initializing test data ...");
|
|
||||||
let mut rng = rand::thread_rng();
|
|
||||||
let keys = generate_random_byte_vector(
|
|
||||||
bench_config.query_count(),
|
|
||||||
bench_config.kvsize(),
|
|
||||||
&mut rng,
|
|
||||||
true,
|
|
||||||
)?;
|
|
||||||
let values = generate_random_byte_vector(
|
|
||||||
bench_config.query_count(),
|
|
||||||
bench_config.kvsize(),
|
|
||||||
&mut rng,
|
|
||||||
false,
|
|
||||||
)?;
|
|
||||||
let new_updated_key = ran_bytes(bench_config.kvsize(), &mut rng);
|
|
||||||
|
|
||||||
// run tests; the idea here is to run all tests one-by-one instead of generating all packets at once
|
|
||||||
// such an approach helps us keep memory usage low
|
|
||||||
// bench set
|
|
||||||
binfo!("Benchmarking SET ...");
|
|
||||||
benches::bench_set(
|
|
||||||
&keys,
|
|
||||||
&values,
|
|
||||||
&mut misc_connection,
|
|
||||||
&bench_config,
|
|
||||||
&switch_table,
|
|
||||||
&mut reports,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// bench update
|
|
||||||
binfo!("Benchmarking UPDATE ...");
|
|
||||||
benches::bench_update(
|
|
||||||
&keys,
|
|
||||||
&new_updated_key,
|
|
||||||
&bench_config,
|
|
||||||
&switch_table,
|
|
||||||
&mut reports,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
// bench get
|
|
||||||
binfo!("Benchmarking GET ...");
|
|
||||||
benches::bench_get(&keys, &bench_config, &switch_table, &mut reports)?;
|
|
||||||
|
|
||||||
// remove all test data
|
|
||||||
binfo!("Finished benchmarks. Cleaning up ...");
|
|
||||||
let r: Element = misc_connection.run_query(Query::from("drop model default.tmpbench force"))?;
|
|
||||||
if r != Element::RespCode(RespCode::Okay) {
|
|
||||||
return Err(Error::Runtime("failed to clean up after benchmarks".into()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if config::should_output_messages() {
|
|
||||||
// normal output
|
|
||||||
println!("===========RESULTS===========");
|
|
||||||
let (maxpad, reports) = reports.finish();
|
|
||||||
for report in reports {
|
|
||||||
let padding = " ".repeat(maxpad - report.name().len());
|
|
||||||
println!(
|
|
||||||
"{}{} {:.6}/sec",
|
|
||||||
report.name().to_uppercase(),
|
|
||||||
padding,
|
|
||||||
report.stat(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
println!("=============================");
|
|
||||||
} else {
|
|
||||||
// JSON
|
|
||||||
println!("{}", reports.into_json())
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@ -1,82 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Wed Aug 10 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use serde::Serialize;
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub struct SingleReport {
|
|
||||||
name: &'static str,
|
|
||||||
stat: f64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SingleReport {
|
|
||||||
pub fn new(name: &'static str, stat: f64) -> Self {
|
|
||||||
Self { name, stat }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn stat(&self) -> f64 {
|
|
||||||
self.stat
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn name(&self) -> &str {
|
|
||||||
self.name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct AggregateReport {
|
|
||||||
names: Vec<SingleReport>,
|
|
||||||
query_count: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AggregateReport {
|
|
||||||
pub fn new(query_count: usize) -> Self {
|
|
||||||
Self {
|
|
||||||
names: Vec::new(),
|
|
||||||
query_count,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn push(&mut self, report: SingleReport) {
|
|
||||||
self.names.push(report)
|
|
||||||
}
|
|
||||||
pub(crate) fn into_json(self) -> String {
|
|
||||||
let (_, report) = self.finish();
|
|
||||||
serde_json::to_string(&report).unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn finish(self) -> (usize, Vec<SingleReport>) {
|
|
||||||
let mut maxpad = self.names[0].name.len();
|
|
||||||
let mut reps = self.names;
|
|
||||||
reps.iter_mut().for_each(|rep| {
|
|
||||||
let total_time = rep.stat;
|
|
||||||
let qps = (self.query_count as f64 / total_time) * 1_000_000_000_f64;
|
|
||||||
rep.stat = qps;
|
|
||||||
if rep.name.len() > maxpad {
|
|
||||||
maxpad = rep.name.len();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
(maxpad, reps)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Tue Aug 09 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
pub const RESPCODE_OKAY: &[u8] = b"*!0\n";
|
|
||||||
|
|
||||||
pub fn calculate_response_size(keylen: usize) -> usize {
|
|
||||||
/*
|
|
||||||
*+5\n
|
|
||||||
hello
|
|
||||||
*/
|
|
||||||
let mut size = 2; // simple query byte + tsymbol
|
|
||||||
size += keylen.to_string().len(); // bytes in length
|
|
||||||
size += 1; // LF
|
|
||||||
size += keylen; // payload
|
|
||||||
size
|
|
||||||
}
|
|
||||||
@ -1,72 +0,0 @@
|
|||||||
#
|
|
||||||
# Created on Tue Nov 03 2020
|
|
||||||
#
|
|
||||||
# This file is a part of Skytable
|
|
||||||
# Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
#
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Affero General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
|
||||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
#
|
|
||||||
|
|
||||||
name: Skytable Benchmark Tool
|
|
||||||
version: 0.8.0
|
|
||||||
author: Sayan N. <ohsayan@outlook.com>
|
|
||||||
about: |
|
|
||||||
The Skytable benchmark tool can be used to benchmark Skytable installations.
|
|
||||||
If you find any issues, then report one here: https://github.com/skytable/skytable
|
|
||||||
args:
|
|
||||||
- connections:
|
|
||||||
short: c
|
|
||||||
long: connections
|
|
||||||
value_name: count
|
|
||||||
help: Sets the number of simultaneous clients
|
|
||||||
takes_value: true
|
|
||||||
- queries:
|
|
||||||
short: q
|
|
||||||
long: queries
|
|
||||||
value_name: number
|
|
||||||
help: Sets the number of queries to run
|
|
||||||
takes_value: true
|
|
||||||
- size:
|
|
||||||
short: s
|
|
||||||
long: kvsize
|
|
||||||
value_name: bytes
|
|
||||||
help: Sets the size of the key/value pairs
|
|
||||||
takes_value: true
|
|
||||||
- json:
|
|
||||||
required: false
|
|
||||||
long: json
|
|
||||||
help: Sets output type to JSON
|
|
||||||
takes_value: false
|
|
||||||
- host:
|
|
||||||
short: h
|
|
||||||
required: false
|
|
||||||
long: host
|
|
||||||
value_name: host
|
|
||||||
help: Sets the remote host to connect to
|
|
||||||
takes_value: true
|
|
||||||
- port:
|
|
||||||
short: p
|
|
||||||
required: false
|
|
||||||
long: port
|
|
||||||
value_name: port
|
|
||||||
help: Sets the remote port to connect to
|
|
||||||
takes_value: true
|
|
||||||
- runs:
|
|
||||||
short: r
|
|
||||||
required: false
|
|
||||||
long: runs
|
|
||||||
value_name: runs
|
|
||||||
takes_value: true
|
|
||||||
help: Sets the number of times the entire test should be run
|
|
||||||
@ -1,148 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Mon Aug 08 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
crate::error::{BResult, Error},
|
|
||||||
crate::util,
|
|
||||||
clap::ArgMatches,
|
|
||||||
std::{fmt::Display, str::FromStr},
|
|
||||||
};
|
|
||||||
|
|
||||||
static mut OUTPUT_JSON: bool = false;
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct ServerConfig {
|
|
||||||
/// host
|
|
||||||
host: Box<str>,
|
|
||||||
/// port
|
|
||||||
port: u16,
|
|
||||||
/// connection count for network pool
|
|
||||||
connections: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
fn try_update<T: FromStr, S: AsRef<str>>(input: Option<S>, target: &mut T) -> BResult<()>
|
|
||||||
where
|
|
||||||
<T as FromStr>::Err: Display,
|
|
||||||
{
|
|
||||||
if let Some(input) = input {
|
|
||||||
let parsed = input
|
|
||||||
.as_ref()
|
|
||||||
.parse::<T>()
|
|
||||||
.map_err(|e| Error::Config(format!("parse error: `{}`", e)))?;
|
|
||||||
*target = parsed;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ServerConfig {
|
|
||||||
const DEFAULT_HOST: &'static str = "127.0.0.1";
|
|
||||||
const DEFAULT_PORT: u16 = 2003;
|
|
||||||
const DEFAULT_CONNECTIONS: usize = 10;
|
|
||||||
/// Init the default server config
|
|
||||||
pub fn new(matches: &ArgMatches) -> BResult<Self> {
|
|
||||||
let mut slf = Self {
|
|
||||||
host: Self::DEFAULT_HOST.into(),
|
|
||||||
port: Self::DEFAULT_PORT,
|
|
||||||
connections: Self::DEFAULT_CONNECTIONS,
|
|
||||||
};
|
|
||||||
slf.try_host(matches.value_of_lossy("host"));
|
|
||||||
slf.try_port(matches.value_of_lossy("port"))?;
|
|
||||||
slf.try_connections(matches.value_of_lossy("connections"))?;
|
|
||||||
Ok(slf)
|
|
||||||
}
|
|
||||||
/// Update the host
|
|
||||||
pub fn try_host<T: AsRef<str>>(&mut self, host: Option<T>) {
|
|
||||||
if let Some(host) = host {
|
|
||||||
self.host = host.as_ref().into();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/// Attempt to update the port
|
|
||||||
pub fn try_port<T: AsRef<str>>(&mut self, port: Option<T>) -> BResult<()> {
|
|
||||||
try_update(port, &mut self.port)
|
|
||||||
}
|
|
||||||
/// Attempt to update the connections
|
|
||||||
pub fn try_connections<T: AsRef<str>>(&mut self, con: Option<T>) -> BResult<()> {
|
|
||||||
try_update(con, &mut self.connections)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ServerConfig {
|
|
||||||
pub fn host(&self) -> &str {
|
|
||||||
self.host.as_ref()
|
|
||||||
}
|
|
||||||
pub fn port(&self) -> u16 {
|
|
||||||
self.port
|
|
||||||
}
|
|
||||||
pub fn connections(&self) -> usize {
|
|
||||||
self.connections
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Benchmark configuration
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct BenchmarkConfig {
|
|
||||||
pub server: ServerConfig,
|
|
||||||
kvsize: usize,
|
|
||||||
queries: usize,
|
|
||||||
runs: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl BenchmarkConfig {
|
|
||||||
const DEFAULT_QUERIES: usize = 100_000;
|
|
||||||
const DEFAULT_KVSIZE: usize = 3;
|
|
||||||
const DEFAULT_RUNS: usize = 5;
|
|
||||||
pub fn new(server: &ServerConfig, matches: ArgMatches) -> BResult<Self> {
|
|
||||||
let mut slf = Self {
|
|
||||||
server: server.clone(),
|
|
||||||
queries: Self::DEFAULT_QUERIES,
|
|
||||||
kvsize: Self::DEFAULT_KVSIZE,
|
|
||||||
runs: Self::DEFAULT_RUNS,
|
|
||||||
};
|
|
||||||
try_update(matches.value_of_lossy("queries"), &mut slf.queries)?;
|
|
||||||
try_update(matches.value_of_lossy("size"), &mut slf.kvsize)?;
|
|
||||||
try_update(matches.value_of_lossy("runs"), &mut slf.runs)?;
|
|
||||||
util::ensure_main_thread();
|
|
||||||
unsafe {
|
|
||||||
OUTPUT_JSON = matches.is_present("json");
|
|
||||||
}
|
|
||||||
Ok(slf)
|
|
||||||
}
|
|
||||||
pub fn kvsize(&self) -> usize {
|
|
||||||
self.kvsize
|
|
||||||
}
|
|
||||||
pub fn query_count(&self) -> usize {
|
|
||||||
self.queries
|
|
||||||
}
|
|
||||||
pub fn runs(&self) -> usize {
|
|
||||||
self.runs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn should_output_messages() -> bool {
|
|
||||||
util::ensure_main_thread();
|
|
||||||
unsafe { !OUTPUT_JSON }
|
|
||||||
}
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Mon Aug 08 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
libstress::WorkpoolError,
|
|
||||||
skytable::error::Error as SkyError,
|
|
||||||
std::{collections::TryReserveError, fmt::Display},
|
|
||||||
};
|
|
||||||
|
|
||||||
pub type BResult<T> = Result<T, Error>;
|
|
||||||
|
|
||||||
/// Benchmark tool errors
|
|
||||||
pub enum Error {
|
|
||||||
/// An error originating from the Skytable client
|
|
||||||
Client(SkyError),
|
|
||||||
/// An error originating from the benchmark/server configuration
|
|
||||||
Config(String),
|
|
||||||
/// A runtime error
|
|
||||||
Runtime(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<SkyError> for Error {
|
|
||||||
fn from(e: SkyError) -> Self {
|
|
||||||
Self::Client(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Display for Error {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
match self {
|
|
||||||
Error::Client(e) => write!(f, "client error: {}", e),
|
|
||||||
Error::Config(e) => write!(f, "config error: {}", e),
|
|
||||||
Error::Runtime(e) => write!(f, "runtime error: {}", e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<TryReserveError> for Error {
|
|
||||||
fn from(e: TryReserveError) -> Self {
|
|
||||||
Error::Runtime(format!("memory reserve error: {}", e))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<WorkpoolError> for Error {
|
|
||||||
fn from(e: WorkpoolError) -> Self {
|
|
||||||
Error::Runtime(format!("threadpool error: {}", e))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,143 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Tue Aug 09 2022
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
crate::{
|
|
||||||
config::ServerConfig,
|
|
||||||
error::{BResult, Error},
|
|
||||||
},
|
|
||||||
skytable::{Connection, Element, Query, RespCode},
|
|
||||||
std::thread,
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Check if the provided keysize has enough combinations to support the given `queries` count
|
|
||||||
///
|
|
||||||
/// This function is heavily optimized and should take Θ(1) time. The `ALWAYS_TRUE_FACTOR` is
|
|
||||||
/// dependent on pointer width (more specifically the virtual address space size).
|
|
||||||
/// - For 64-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 12, we'll hit the maximum
|
|
||||||
/// of the address space and hence this will always return true (because of the size of `usize`)
|
|
||||||
/// > The value for r = 12 is `1.27309515e+20` which largely exceeds `1.8446744e+19`
|
|
||||||
/// - For 32-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 5, we'll hit the maximum
|
|
||||||
/// of the address space and hence this will always return true (because of the size of `usize`)
|
|
||||||
/// > The value for r = 5 is `8.81e+9` which largely exceeds `4.3e+9`
|
|
||||||
pub const fn has_enough_ncr(keysize: usize, queries: usize) -> bool {
|
|
||||||
const LUT: [u64; 11] = [
|
|
||||||
// 1B
|
|
||||||
256,
|
|
||||||
// 2B
|
|
||||||
32640,
|
|
||||||
// 3B
|
|
||||||
2763520,
|
|
||||||
// 4B
|
|
||||||
174792640,
|
|
||||||
// 5B
|
|
||||||
8809549056,
|
|
||||||
// 6B
|
|
||||||
368532802176,
|
|
||||||
// 7B
|
|
||||||
13161885792000,
|
|
||||||
// 8B
|
|
||||||
409663695276000,
|
|
||||||
// 9B
|
|
||||||
11288510714272000,
|
|
||||||
// 10B
|
|
||||||
278826214642518400,
|
|
||||||
// 11B
|
|
||||||
6235568072914502400,
|
|
||||||
];
|
|
||||||
#[cfg(target_pointer_width = "64")]
|
|
||||||
const ALWAYS_TRUE_FACTOR: usize = 12;
|
|
||||||
#[cfg(target_pointer_width = "32")]
|
|
||||||
const ALWAYS_TRUE_FACTOR: usize = 5;
|
|
||||||
keysize >= ALWAYS_TRUE_FACTOR || (LUT[keysize - 1] >= queries as _)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Run a sanity test, making sure that the server is ready for benchmarking. This function will do the
|
|
||||||
/// following tests:
|
|
||||||
/// - Connect to the instance
|
|
||||||
/// - Run a `heya` as a preliminary test
|
|
||||||
/// - Create a new table `tmpbench`. This is where we're supposed to run all the benchmarks.
|
|
||||||
/// - Switch to the new table
|
|
||||||
/// - Set a key, and get it checking the equality of the returned value
|
|
||||||
pub fn run_sanity_test(server_config: &ServerConfig) -> BResult<()> {
|
|
||||||
let mut con = Connection::new(server_config.host(), server_config.port())?;
|
|
||||||
let tests: [(Query, Element, &str); 5] = [
|
|
||||||
(
|
|
||||||
Query::from("HEYA"),
|
|
||||||
Element::String("HEY!".to_owned()),
|
|
||||||
"heya",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
Query::from("CREATE MODEL default.tmpbench(binary, binary)"),
|
|
||||||
Element::RespCode(RespCode::Okay),
|
|
||||||
"create model",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
Query::from("use default.tmpbench"),
|
|
||||||
Element::RespCode(RespCode::Okay),
|
|
||||||
"use",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
Query::from("set").arg("x").arg("100"),
|
|
||||||
Element::RespCode(RespCode::Okay),
|
|
||||||
"set",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
Query::from("get").arg("x"),
|
|
||||||
Element::Binstr("100".as_bytes().to_owned()),
|
|
||||||
"get",
|
|
||||||
),
|
|
||||||
];
|
|
||||||
for (query, expected, test_kind) in tests {
|
|
||||||
let r: Element = con.run_query(query)?;
|
|
||||||
if r != expected {
|
|
||||||
return Err(Error::Runtime(format!(
|
|
||||||
"sanity test for `{test_kind}` failed"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ensures that the current thread is the main thread. If not, this function will panic
|
|
||||||
pub fn ensure_main_thread() {
|
|
||||||
assert_eq!(
|
|
||||||
thread::current().name().unwrap(),
|
|
||||||
"main",
|
|
||||||
"unsafe function called from non-main thread"
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Run a cleanup. This function attempts to remove the `default.tmpbench` entity
|
|
||||||
pub fn cleanup(server_config: &ServerConfig) -> BResult<()> {
|
|
||||||
let mut c = Connection::new(server_config.host(), server_config.port())?;
|
|
||||||
let r: Element = c.run_query(Query::from("drop model default.tmpbench force"))?;
|
|
||||||
if r == Element::RespCode(RespCode::Okay) {
|
|
||||||
Err(Error::Runtime("failed to run cleanup".into()))
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "sky-migrate"
|
|
||||||
version = "0.8.0"
|
|
||||||
authors = ["Sayan Nandan <nandansayan@outlook.com>"]
|
|
||||||
edition = "2021"
|
|
||||||
|
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
skytable = { git = "https://github.com/skytable/client-rust.git" }
|
|
||||||
env_logger = "0.10.0"
|
|
||||||
bincode = "1.3.3"
|
|
||||||
log = "0.4.19"
|
|
||||||
clap = { version = "2", features = ["yaml"] }
|
|
||||||
@ -1,10 +0,0 @@
|
|||||||
# Skytable migration tool
|
|
||||||
|
|
||||||
The Skytable migration tool can be used to perform migrations between database versions. The basic idea is:
|
|
||||||
"read all data from the older machine and send it over to the newer one."
|
|
||||||
|
|
||||||
For migrating versions from 0.6 to 0.7, the database administrator has to launch the tool in the old data directory of the old instance and then pass the new instance host/port information. The tool will then read the data from the directory (this was possible because 0.6 used a very simple disk format than newer versions). This approach however has the advantage of not having to start the database server for the migration to happen.
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).
|
|
||||||
@ -1,30 +0,0 @@
|
|||||||
name: Skytable Migration Tool
|
|
||||||
version: 0.7.0
|
|
||||||
author: Sayan N. <ohsayan@outlook.com>
|
|
||||||
about: |
|
|
||||||
The Skytable migration tool allows users coming from older versions (>=0.8.0)
|
|
||||||
to upgrade their datasets to the latest Skytable version. This tool currently
|
|
||||||
supports versions >= 0.8.0 and upgrading it to 0.7.0. To upgrade, on needs
|
|
||||||
to simply run:
|
|
||||||
sky-migrate --prevdir <lastpath> --new <host>:<port>
|
|
||||||
Where `<lastpath>` is the path to the last installation's data directory and
|
|
||||||
`<host>` and `<port>` is the hostname and port for the new server instance
|
|
||||||
args:
|
|
||||||
- new:
|
|
||||||
long: new
|
|
||||||
takes_value: true
|
|
||||||
required: true
|
|
||||||
help: The <host>:<port> combo for the new instance
|
|
||||||
value_name: new
|
|
||||||
- prevdir:
|
|
||||||
long: prevdir
|
|
||||||
takes_value: true
|
|
||||||
required: true
|
|
||||||
help: Path to the previous installation location
|
|
||||||
value_name: prevdir
|
|
||||||
- serial:
|
|
||||||
long: serial
|
|
||||||
takes_value: false
|
|
||||||
required: false
|
|
||||||
help: |
|
|
||||||
Transfer entries one-by-one instead of all at once to save memory
|
|
||||||
@ -1,129 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Tue Aug 17 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
#![allow(clippy::unit_arg)]
|
|
||||||
|
|
||||||
use {
|
|
||||||
clap::{load_yaml, App},
|
|
||||||
core::hint::unreachable_unchecked,
|
|
||||||
env_logger::Builder,
|
|
||||||
log::{error as err, info},
|
|
||||||
skytable::{query, sync::Connection, Element, Query, RespCode},
|
|
||||||
std::{collections::HashMap, env, fs, path::PathBuf, process},
|
|
||||||
};
|
|
||||||
|
|
||||||
type Bytes = Vec<u8>;
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
// first evaluate config
|
|
||||||
let cfg_layout = load_yaml!("cli.yml");
|
|
||||||
let matches = App::from_yaml(cfg_layout).get_matches();
|
|
||||||
Builder::new()
|
|
||||||
.parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned()))
|
|
||||||
.init();
|
|
||||||
let new_host = matches
|
|
||||||
.value_of("new")
|
|
||||||
.map(|v| v.to_string())
|
|
||||||
.unwrap_or_else(|| unsafe { unreachable_unchecked() });
|
|
||||||
let serial = matches.is_present("serial");
|
|
||||||
let hostsplit: Vec<&str> = new_host.split(':').collect();
|
|
||||||
if hostsplit.len() != 2 {
|
|
||||||
err(err!("Bad value for --new"));
|
|
||||||
}
|
|
||||||
let (host, port) = unsafe { (hostsplit.get_unchecked(0), hostsplit.get_unchecked(1)) };
|
|
||||||
let port = match port.parse() {
|
|
||||||
Ok(p) => p,
|
|
||||||
Err(e) => err(err!("Bad value for port in --new: {}", e)),
|
|
||||||
};
|
|
||||||
let mut old_dir = matches
|
|
||||||
.value_of("prevdir")
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|| unsafe { unreachable_unchecked() });
|
|
||||||
old_dir.push("data.bin");
|
|
||||||
// now connect
|
|
||||||
let mut con = match Connection::new(host, port) {
|
|
||||||
Ok(con) => con,
|
|
||||||
Err(e) => err(err!("Failed to connect to new instance with error: {}", e)),
|
|
||||||
};
|
|
||||||
// run sanity test
|
|
||||||
let q = query!("HEYA");
|
|
||||||
match con.run_query_raw(&q) {
|
|
||||||
Ok(Element::String(s)) if s.eq("HEY!") => {}
|
|
||||||
Ok(_) => err(err!("Unknown response from server")),
|
|
||||||
Err(e) => err(err!(
|
|
||||||
"An I/O error occurred while running sanity test: {}",
|
|
||||||
e
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
info!("Sanity test complete");
|
|
||||||
|
|
||||||
// now de old file
|
|
||||||
let read = match fs::read(old_dir) {
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(e) => err(err!(
|
|
||||||
"Failed to read data.bin file from old directory: {}",
|
|
||||||
e
|
|
||||||
)),
|
|
||||||
};
|
|
||||||
let de: HashMap<Bytes, Bytes> = match bincode::deserialize(&read) {
|
|
||||||
Ok(r) => r,
|
|
||||||
Err(e) => err(err!("Failed to unpack old file with: {}", e)),
|
|
||||||
};
|
|
||||||
unsafe {
|
|
||||||
if serial {
|
|
||||||
// transfer serially
|
|
||||||
for (key, value) in de.into_iter() {
|
|
||||||
let q = query!(
|
|
||||||
"USET",
|
|
||||||
String::from_utf8_unchecked(key),
|
|
||||||
String::from_utf8_unchecked(value)
|
|
||||||
);
|
|
||||||
okay(&mut con, q)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// transfer all at once
|
|
||||||
let mut query = Query::from("USET");
|
|
||||||
for (key, value) in de.into_iter() {
|
|
||||||
query.push(String::from_utf8_unchecked(key));
|
|
||||||
query.push(String::from_utf8_unchecked(value));
|
|
||||||
}
|
|
||||||
okay(&mut con, query)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
info!("Finished migration");
|
|
||||||
}
|
|
||||||
|
|
||||||
fn err(_i: ()) -> ! {
|
|
||||||
process::exit(0x01)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn okay(con: &mut Connection, q: Query) {
|
|
||||||
match con.run_query_raw(&q) {
|
|
||||||
Ok(Element::RespCode(RespCode::Okay)) => {}
|
|
||||||
Err(e) => err(err!("An I/O error occurred while running query: {}", e)),
|
|
||||||
Ok(_) => err(err!("Unknown response from server")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "stress-test"
|
|
||||||
version = "0.1.0"
|
|
||||||
authors = ["Sayan Nandan <nandansayan@outlook.com>"]
|
|
||||||
edition = "2021"
|
|
||||||
|
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
# internal deps
|
|
||||||
libstress = { path = "../libstress" }
|
|
||||||
skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next", features = [
|
|
||||||
"dbg",
|
|
||||||
] }
|
|
||||||
devtimer = "4.0.1"
|
|
||||||
# external deps
|
|
||||||
sysinfo = "0.29.7"
|
|
||||||
env_logger = "0.10.0"
|
|
||||||
log = "0.4.19"
|
|
||||||
rand = "0.8.5"
|
|
||||||
crossbeam-channel = "0.5.8"
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
# Skytable stress-test tool
|
|
||||||
|
|
||||||
This is a tool in its infancy, but its ultimate goal is to provide a stress testing framework for Skytable. For now, it tests linearity (core scalability) with increasing clients.
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).
|
|
||||||
@ -1,264 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Fri Jun 18 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! # Client linearity tests
|
|
||||||
//!
|
|
||||||
//! This module contains functions to test the linearity of the database with increasing number
|
|
||||||
//! of clients, i.e how the number of queries scale with increasing clients. These functions
|
|
||||||
//! however, DO NOT focus on benchmarking and instead focus on correctness under load from
|
|
||||||
//! concurrent clients.
|
|
||||||
//!
|
|
||||||
|
|
||||||
use {
|
|
||||||
crate::{logstress, DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV},
|
|
||||||
crossbeam_channel::bounded,
|
|
||||||
devtimer::SimpleTimer,
|
|
||||||
libstress::{rayon::prelude::*, utils::generate_random_string_vector, Workpool},
|
|
||||||
skytable::{actions::Actions, query, Connection, Element, Query, RespCode},
|
|
||||||
};
|
|
||||||
|
|
||||||
macro_rules! log_client_linearity {
|
|
||||||
($stressid:expr, $counter:expr, $what:expr) => {
|
|
||||||
log::info!(
|
|
||||||
"Stress ({}{}) [{}]: Clients: {}; K/V size: {}; Queries: {}",
|
|
||||||
$stressid,
|
|
||||||
$counter,
|
|
||||||
$what,
|
|
||||||
$counter,
|
|
||||||
DEFAULT_SIZE_KV,
|
|
||||||
DEFAULT_QUERY_COUNT
|
|
||||||
);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// This object provides methods to measure the percentage change in the slope
|
|
||||||
/// of a function that is expected to have linearity.
|
|
||||||
///
|
|
||||||
/// For example, we can think of it to work in the following way:
|
|
||||||
/// Let h(x) be a function with linearity (not proportionality because that isn't
|
|
||||||
/// applicable in our case). h(x) gives us the time taken to run a given number of
|
|
||||||
/// queries (invariant; not plotted on axes), where x is the number of concurrent
|
|
||||||
/// clients. As we would want our database to scale with increasing clients (and cores),
|
|
||||||
/// we'd expect linearity, hence the gradient should continue to fall with increasing
|
|
||||||
/// values in the +ve x-axis effectively producing a constantly decreasing slope, reflected
|
|
||||||
/// by increasing values of abs(get_delta(h(x))).
|
|
||||||
///
|
|
||||||
/// TODO(@ohsayan): Of course, some unexpected kernel errors/scheduler hiccups et al can
|
|
||||||
/// cause there to be a certain epsilon that must be tolerated with a tolerance factor
|
|
||||||
///
|
|
||||||
pub struct LinearityMeter {
|
|
||||||
init: Option<u128>,
|
|
||||||
measure: Vec<f32>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LinearityMeter {
|
|
||||||
pub const fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
init: None,
|
|
||||||
measure: Vec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
pub fn get_delta(&mut self, current: u128) -> f32 {
|
|
||||||
if let Some(u) = self.init {
|
|
||||||
let cur = ((current as f32 - u as f32) / u as f32) * 100.00_f32;
|
|
||||||
self.measure.push(cur);
|
|
||||||
cur
|
|
||||||
} else {
|
|
||||||
// if init is not initialized, initialize it
|
|
||||||
self.init = Some(current);
|
|
||||||
// no change when at base
|
|
||||||
0.00
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn stress_linearity_concurrent_clients_set(
|
|
||||||
mut rng: &mut impl rand::Rng,
|
|
||||||
max_workers: usize,
|
|
||||||
temp_con: &mut Connection,
|
|
||||||
) {
|
|
||||||
logstress!(
|
|
||||||
"A [SET]",
|
|
||||||
"Linearity test with monotonically increasing clients"
|
|
||||||
);
|
|
||||||
let mut current_thread_count = 1usize;
|
|
||||||
|
|
||||||
// generate the random k/v pairs
|
|
||||||
let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true);
|
|
||||||
let values =
|
|
||||||
generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false);
|
|
||||||
let (keys, values) = match (keys, values) {
|
|
||||||
(Ok(k), Ok(v)) => (k, v),
|
|
||||||
_ => {
|
|
||||||
eprintln!("Allocation error");
|
|
||||||
std::process::exit(0x01);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
// make sure the database is empty
|
|
||||||
temp_con.flushdb().unwrap();
|
|
||||||
|
|
||||||
// initialize the linearity counter
|
|
||||||
let mut linearity = LinearityMeter::new();
|
|
||||||
while current_thread_count <= max_workers {
|
|
||||||
log_client_linearity!("A", current_thread_count, "SET");
|
|
||||||
// generate the set packets
|
|
||||||
let set_packs: Vec<Query> = keys
|
|
||||||
.par_iter()
|
|
||||||
.zip(values.par_iter())
|
|
||||||
.map(|(k, v)| query!("SET", k, v))
|
|
||||||
.collect();
|
|
||||||
let workpool = Workpool::new(
|
|
||||||
current_thread_count,
|
|
||||||
|| Connection::new("127.0.0.1", 2003).unwrap(),
|
|
||||||
move |sock, query| {
|
|
||||||
assert_eq!(
|
|
||||||
sock.run_query_raw(&query).unwrap(),
|
|
||||||
Element::RespCode(RespCode::Okay)
|
|
||||||
);
|
|
||||||
},
|
|
||||||
|_| {},
|
|
||||||
true,
|
|
||||||
Some(DEFAULT_QUERY_COUNT),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let mut timer = SimpleTimer::new();
|
|
||||||
timer.start();
|
|
||||||
workpool.execute_and_finish_iter(set_packs);
|
|
||||||
timer.stop();
|
|
||||||
log::info!(
|
|
||||||
"Delta: {}%",
|
|
||||||
linearity.get_delta(timer.time_in_nanos().unwrap())
|
|
||||||
);
|
|
||||||
// clean up the database
|
|
||||||
temp_con.flushdb().unwrap();
|
|
||||||
current_thread_count += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn stress_linearity_concurrent_clients_get(
|
|
||||||
mut rng: &mut impl rand::Rng,
|
|
||||||
max_workers: usize,
|
|
||||||
temp_con: &mut Connection,
|
|
||||||
) {
|
|
||||||
logstress!(
|
|
||||||
"A [GET]",
|
|
||||||
"Linearity test with monotonically increasing clients"
|
|
||||||
);
|
|
||||||
let mut current_thread_count = 1usize;
|
|
||||||
|
|
||||||
// generate the random k/v pairs
|
|
||||||
let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true);
|
|
||||||
let values =
|
|
||||||
generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false);
|
|
||||||
let (keys, values) = match (keys, values) {
|
|
||||||
(Ok(k), Ok(v)) => (k, v),
|
|
||||||
_ => {
|
|
||||||
eprintln!("Allocation error");
|
|
||||||
std::process::exit(0x01);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Make sure that the database is empty
|
|
||||||
temp_con.flushdb().unwrap();
|
|
||||||
|
|
||||||
// First set the keys
|
|
||||||
let set_packs: Vec<Query> = keys
|
|
||||||
.par_iter()
|
|
||||||
.zip(values.par_iter())
|
|
||||||
.map(|(k, v)| query!("SET", k, v))
|
|
||||||
.collect();
|
|
||||||
let workpool = Workpool::new_default_threads(
|
|
||||||
|| Connection::new("127.0.0.1", 2003).unwrap(),
|
|
||||||
move |sock, query| {
|
|
||||||
assert_eq!(
|
|
||||||
sock.run_query_raw(&query).unwrap(),
|
|
||||||
Element::RespCode(RespCode::Okay)
|
|
||||||
);
|
|
||||||
},
|
|
||||||
|_| {},
|
|
||||||
true,
|
|
||||||
Some(DEFAULT_QUERY_COUNT),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
workpool.execute_and_finish_iter(set_packs);
|
|
||||||
|
|
||||||
// initialize the linearity counter
|
|
||||||
let mut linearity = LinearityMeter::new();
|
|
||||||
while current_thread_count <= max_workers {
|
|
||||||
log_client_linearity!("A", current_thread_count, "GET");
|
|
||||||
/*
|
|
||||||
We create a mpmc to receive the results returned. This avoids us using
|
|
||||||
any kind of locking on the surface which can slow down things
|
|
||||||
*/
|
|
||||||
let (tx, rx) = bounded::<Element>(DEFAULT_QUERY_COUNT);
|
|
||||||
|
|
||||||
// generate the get packets
|
|
||||||
let get_packs: Vec<Query> = keys.iter().map(|k| query!("GET", k)).collect();
|
|
||||||
let wp = Workpool::new(
|
|
||||||
current_thread_count,
|
|
||||||
|| Connection::new("127.0.0.1", 2003).unwrap(),
|
|
||||||
move |sock, query| {
|
|
||||||
let tx = tx.clone();
|
|
||||||
tx.send(sock.run_query_raw(&query).unwrap()).unwrap();
|
|
||||||
},
|
|
||||||
|_| {},
|
|
||||||
true,
|
|
||||||
Some(DEFAULT_QUERY_COUNT),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let mut timer = SimpleTimer::new();
|
|
||||||
timer.start();
|
|
||||||
wp.execute_and_finish_iter(get_packs);
|
|
||||||
timer.stop();
|
|
||||||
log::info!(
|
|
||||||
"Delta: {}%",
|
|
||||||
linearity.get_delta(timer.time_in_nanos().unwrap())
|
|
||||||
);
|
|
||||||
let rets: Vec<String> = rx
|
|
||||||
.into_iter()
|
|
||||||
.map(|v| {
|
|
||||||
if let Element::String(val) = v {
|
|
||||||
val
|
|
||||||
} else {
|
|
||||||
panic!("Unexpected response from server");
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
assert_eq!(
|
|
||||||
rets.len(),
|
|
||||||
values.len(),
|
|
||||||
"Incorrect number of values returned by server"
|
|
||||||
);
|
|
||||||
|
|
||||||
// now evaluate them
|
|
||||||
assert!(
|
|
||||||
rets.into_par_iter().all(|v| values.contains(&v)),
|
|
||||||
"Values returned by the server don't match what was sent"
|
|
||||||
);
|
|
||||||
current_thread_count += 1;
|
|
||||||
}
|
|
||||||
temp_con.flushdb().unwrap();
|
|
||||||
}
|
|
||||||
@ -1,90 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Wed Jun 16 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
#![deny(unused_crate_dependencies)]
|
|
||||||
#![deny(unused_imports)]
|
|
||||||
|
|
||||||
use std::thread::available_parallelism;
|
|
||||||
|
|
||||||
use {
|
|
||||||
libstress::traits::ExitError,
|
|
||||||
log::{info, trace, warn},
|
|
||||||
rand::thread_rng,
|
|
||||||
skytable::Connection,
|
|
||||||
std::env,
|
|
||||||
sysinfo::{RefreshKind, System, SystemExt},
|
|
||||||
};
|
|
||||||
mod linearity_client;
|
|
||||||
mod utils;
|
|
||||||
|
|
||||||
pub const DEFAULT_SIZE_KV: usize = 4;
|
|
||||||
pub const DEFAULT_QUERY_COUNT: usize = 100_000_usize;
|
|
||||||
|
|
||||||
#[macro_export]
|
|
||||||
macro_rules! logstress {
|
|
||||||
($stressid:expr, $extra:expr) => {
|
|
||||||
log::info!("Stress ({}): {}", $stressid, $extra);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
// Build the logger
|
|
||||||
env_logger::Builder::new()
|
|
||||||
.parse_filters(&env::var("SKY_STRESS_LOG").unwrap_or_else(|_| "trace".to_owned()))
|
|
||||||
.init();
|
|
||||||
warn!("The stress test checks correctness under load and DOES NOT show the true throughput");
|
|
||||||
|
|
||||||
// get the rng and refresh sysinfo
|
|
||||||
let mut rng = thread_rng();
|
|
||||||
// we only need to refresh memory and CPU info; don't waste time syncing other things
|
|
||||||
let to_refresh = RefreshKind::new().with_memory();
|
|
||||||
let mut sys = System::new_with_specifics(to_refresh);
|
|
||||||
sys.refresh_specifics(to_refresh);
|
|
||||||
let core_count = available_parallelism().map_or(1, usize::from);
|
|
||||||
let max_workers = core_count * 2;
|
|
||||||
trace!(
|
|
||||||
"This host has {} logical cores. Will spawn a maximum of {} threads",
|
|
||||||
core_count,
|
|
||||||
max_workers * 2
|
|
||||||
);
|
|
||||||
|
|
||||||
// establish a connection to ensure sanity
|
|
||||||
let mut temp_con = Connection::new("127.0.0.1", 2003).exit_error("Failed to connect to server");
|
|
||||||
|
|
||||||
// calculate the maximum keylen
|
|
||||||
let max_keylen = utils::calculate_max_keylen(DEFAULT_QUERY_COUNT, &mut sys);
|
|
||||||
info!(
|
|
||||||
"This host can support a maximum theoretical keylen of: {}",
|
|
||||||
max_keylen
|
|
||||||
);
|
|
||||||
|
|
||||||
// run the actual stress tests
|
|
||||||
linearity_client::stress_linearity_concurrent_clients_set(&mut rng, max_workers, &mut temp_con);
|
|
||||||
linearity_client::stress_linearity_concurrent_clients_get(&mut rng, max_workers, &mut temp_con);
|
|
||||||
|
|
||||||
// done, exit
|
|
||||||
info!("SUCCESS. Stress test complete!");
|
|
||||||
}
|
|
||||||
@ -1,62 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Fri Jun 18 2021
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
use {
|
|
||||||
log::trace,
|
|
||||||
skytable::Query,
|
|
||||||
sysinfo::{System, SystemExt},
|
|
||||||
};
|
|
||||||
|
|
||||||
pub fn calculate_max_keylen(expected_queries: usize, sys: &mut System) -> usize {
|
|
||||||
let total_mem_in_bytes = (sys.total_memory() * 1024) as usize;
|
|
||||||
trace!(
|
|
||||||
"This host has a total memory of: {} Bytes",
|
|
||||||
total_mem_in_bytes
|
|
||||||
);
|
|
||||||
// av_mem gives us 90% of the memory size
|
|
||||||
let ninety_percent_of_memory = (0.90_f32 * total_mem_in_bytes as f32) as usize;
|
|
||||||
let mut highest_len = 1usize;
|
|
||||||
loop {
|
|
||||||
let set_pack_len = Query::array_packet_size_hint(vec![3, highest_len, highest_len]);
|
|
||||||
let get_pack_len = Query::array_packet_size_hint(vec![3, highest_len]);
|
|
||||||
let resulting_size = expected_queries
|
|
||||||
* (
|
|
||||||
// for the set packets
|
|
||||||
set_pack_len +
|
|
||||||
// for the get packets
|
|
||||||
get_pack_len +
|
|
||||||
// for the keys themselves
|
|
||||||
highest_len
|
|
||||||
);
|
|
||||||
if resulting_size >= ninety_percent_of_memory as usize {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
// increase the length by 5% every time to get the maximum possible length
|
|
||||||
// now this 5% increment is a tradeoff, but it's worth it to not wait for
|
|
||||||
// so long
|
|
||||||
highest_len = (highest_len as f32 * 1.05_f32).ceil() as usize;
|
|
||||||
}
|
|
||||||
highest_len
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue