Simplify command assembly

next
Sayan Nandan 3 years ago
parent 7bf21b3880
commit cd1accd646
No known key found for this signature in database
GPG Key ID: 8BC07A0A4D41DD52

@ -26,6 +26,8 @@
use crate::util;
use crate::HarnessResult;
#[cfg(windows)]
use std::os::windows::process::CommandExt;
use std::{path::PathBuf, process::Command};
use zip::CompressionMethod;

@ -0,0 +1,185 @@
/*
* Created on Thu Mar 17 2022
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
* NoSQL database written by Sayan Nandan ("the Author") with the
* vision to provide flexibility in data modelling without compromising
* on performance, queryability or scalability.
*
* Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
use crate::{
build::BuildMode,
util::{self, SLEEP_FOR_TERMINATION},
HarnessError, HarnessResult,
};
use openssl::{
asn1::Asn1Time,
bn::{BigNum, MsbOption},
error::ErrorStack,
hash::MessageDigest,
pkey::{PKey, Private},
rsa::Rsa,
x509::{
extension::{BasicConstraints, KeyUsage, SubjectKeyIdentifier},
X509NameBuilder, X509,
},
};
use std::{fs, io::Write};
mod svc;
/// Run the test suite
pub fn run_test() -> HarnessResult<()> {
let ret = run_test_inner();
let kill_check = svc::kill_servers();
util::sleep_sec(SLEEP_FOR_TERMINATION);
if let Err(e) = kill_check {
error!("Failed to kill servers with error: {e}");
}
// clean up
info!("Cleaning up test directories ...");
fs::remove_dir_all("server1").map_err(|e| {
HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}"))
})?;
fs::remove_dir_all("server2").map_err(|e| {
HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}"))
})?;
ret
}
fn append_target(args: &mut Vec<String>) {
match util::get_var(util::VAR_TARGET) {
Some(target) => {
args.push("--target".into());
args.push(target);
}
None => {}
}
}
/// Actually run the tests. This will run:
/// - The standard test suite
/// - The persistence test suite
fn run_test_inner() -> HarnessResult<()> {
// first create the TLS keys
info!("Creating TLS key+cert");
let (cert, pkey) = mk_ca_cert().expect("Failed to create cert");
let mut certfile = fs::File::create("cert.pem").expect("failed to create cert.pem");
certfile.write_all(&cert.to_pem().unwrap()).unwrap();
let mut pkeyfile = fs::File::create("key.pem").expect("failed to create key.pem");
pkeyfile
.write_all(&pkey.private_key_to_pem_pkcs8().unwrap())
.unwrap();
fs::create_dir_all("server1").map_err(|e| {
HarnessError::Other(format!("Failed to create `server1` dir with error: {e}"))
})?;
fs::create_dir_all("server2").map_err(|e| {
HarnessError::Other(format!("Failed to create `server2` dir with error: {e}"))
})?;
// assemble commands
let target_folder = util::get_target_folder(BuildMode::Debug);
let mut standard_test_suite_args = vec!["cargo".to_owned(), "test".into()];
let mut build_cmd_args = vec![
"cargo".to_owned(),
"build".into(),
"-p".to_owned(),
"skyd".into(),
];
append_target(&mut build_cmd_args);
append_target(&mut standard_test_suite_args);
let persist_test_suite_args = [
standard_test_suite_args.as_slice(),
&["--features".to_owned(), "persist-suite".into()],
]
.concat();
// get cmd
let build_cmd = util::assemble_command_from_slice(build_cmd_args);
let standard_test_suite = util::assemble_command_from_slice(standard_test_suite_args);
let persist_test_suite = util::assemble_command_from_slice(persist_test_suite_args);
// build skyd
info!("Building server binary ...");
util::handle_child("build skyd", build_cmd)?;
// run standard test suite
svc::run_with_servers(&target_folder, true, move || {
info!("Running standard test suite ...");
util::handle_child("standard test suite", standard_test_suite)?;
Ok(())
})?;
// run persistence tests; don't kill the servers because if either of the tests fail
// then we can ensure that they will be killed by `run_test`
svc::run_with_servers(&target_folder, false, move || {
info!("Running persistence test suite ...");
util::handle_child("standard test suite", persist_test_suite)?;
Ok(())
})?;
Ok(())
}
/// Generate certificates
fn mk_ca_cert() -> Result<(X509, PKey<Private>), ErrorStack> {
let rsa = Rsa::generate(2048)?;
let key_pair = PKey::from_rsa(rsa)?;
let mut x509_name = X509NameBuilder::new()?;
x509_name.append_entry_by_text("C", "US")?;
x509_name.append_entry_by_text("ST", "CA")?;
x509_name.append_entry_by_text("O", "Skytable")?;
x509_name.append_entry_by_text("CN", "sky-harness")?;
let x509_name = x509_name.build();
let mut cert_builder = X509::builder()?;
cert_builder.set_version(2)?;
let serial_number = {
let mut serial = BigNum::new()?;
serial.rand(159, MsbOption::MAYBE_ZERO, false)?;
serial.to_asn1_integer()?
};
cert_builder.set_serial_number(&serial_number)?;
cert_builder.set_subject_name(&x509_name)?;
cert_builder.set_issuer_name(&x509_name)?;
cert_builder.set_pubkey(&key_pair)?;
let not_before = Asn1Time::days_from_now(0)?;
cert_builder.set_not_before(&not_before)?;
let not_after = Asn1Time::days_from_now(365)?;
cert_builder.set_not_after(&not_after)?;
cert_builder.append_extension(BasicConstraints::new().critical().ca().build()?)?;
cert_builder.append_extension(
KeyUsage::new()
.critical()
.key_cert_sign()
.crl_sign()
.build()?,
)?;
let subject_key_identifier =
SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?;
cert_builder.append_extension(subject_key_identifier)?;
cert_builder.sign(&key_pair, MessageDigest::sha256())?;
let cert = cert_builder.build();
Ok((cert, key_pair))
}

@ -1,5 +1,5 @@
/*
* Created on Thu Mar 17 2022
* Created on Thu Mar 24 2022
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@ -25,37 +25,18 @@
*/
use crate::{
build::BuildMode,
util::{self, SLEEP_FOR_TERMINATION},
HarnessError, HarnessResult,
};
use openssl::{
asn1::Asn1Time,
bn::{BigNum, MsbOption},
error::ErrorStack,
hash::MessageDigest,
pkey::{PKey, Private},
rsa::Rsa,
x509::{
extension::{BasicConstraints, KeyUsage, SubjectKeyIdentifier},
X509NameBuilder, X509,
},
};
use skytable::Connection;
#[cfg(windows)]
use std::os::windows::process::CommandExt;
use std::{
fs,
io::{Error as IoError, ErrorKind, Write},
path::{Path, PathBuf},
process::{Child, Command, Stdio},
io::{Error as IoError, ErrorKind},
path::Path,
process::{Child, Command},
};
/// The workspace root
const WORKSPACE_ROOT: &str = env!("ROOT_DIR");
#[cfg(windows)]
/// The powershell script hack to send CTRL+C using kernel32
const POWERSHELL_SCRIPT: &str = include_str!("../../ci/windows/stop.ps1");
const POWERSHELL_SCRIPT: &str = include_str!("../../../ci/windows/stop.ps1");
#[cfg(windows)]
/// Flag for new console Window
const CREATE_NEW_CONSOLE: u32 = 0x00000010;
@ -65,19 +46,22 @@ const TESTSUITE_SERVER_HOST: &str = "127.0.0.1";
const TESTSUITE_SERVER_PORTS: [u16; 4] = [2003, 2004, 2005, 2006];
/// The server IDs matching with the configuration files
const SERVER_IDS: [&str; 2] = ["server1", "server2"];
/// The workspace root
const WORKSPACE_ROOT: &str = env!("ROOT_DIR");
/// Get the command to start the provided server1
pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef<Path>) -> Command {
let cfg_file_path = PathBuf::from(format!("{WORKSPACE_ROOT}ci/{server_id}.toml"));
let binpath = util::concat_path("skyd", target_folder)
.to_string_lossy()
.to_string();
let mut cmd = Command::new(binpath);
cmd.arg("--withconfig");
cmd.arg(cfg_file_path);
let args = vec![
// binary
util::concat_path("skyd", target_folder)
.to_string_lossy()
.to_string(),
// config
"--withconfig".to_owned(),
format!("{WORKSPACE_ROOT}ci/{server_id}.toml"),
];
let mut cmd = util::assemble_command_from_slice(&args);
cmd.current_dir(server_id);
cmd.stderr(Stdio::piped());
cmd.stdout(Stdio::piped());
#[cfg(windows)]
cmd.creation_flags(CREATE_NEW_CONSOLE);
cmd
@ -170,26 +154,28 @@ fn start_servers(target_folder: impl AsRef<Path>) -> HarnessResult<Vec<Child>> {
Ok(ret)
}
fn run_with_servers(
pub(super) fn run_with_servers(
target_folder: impl AsRef<Path>,
kill_servers_when_done: bool,
run_what: impl FnOnce() -> HarnessResult<()>,
) -> HarnessResult<()> {
info!("Starting servers ...");
let children = start_servers(target_folder.as_ref())?;
run_what()?;
info!("Terminating server instances ...");
kill_servers()?;
info!("Sent termination signals");
wait_for_shutdown()?;
info!("Terminated server instances");
if kill_servers_when_done {
kill_servers()?;
wait_for_shutdown()?;
}
// just use this to avoid ignoring the children vector
assert_eq!(children.len(), SERVER_IDS.len());
Ok(())
}
/// Send termination signal to the servers
fn kill_servers() -> HarnessResult<()> {
pub(super) fn kill_servers() -> HarnessResult<()> {
info!("Terminating server instances ...");
kill_servers_inner()?;
info!("Sent termination signals");
Ok(())
}
@ -210,134 +196,3 @@ fn kill_servers_inner() -> HarnessResult<()> {
))),
}
}
/// Run the test suite
pub fn run_test() -> HarnessResult<()> {
let ret = run_test_inner();
if let Err(e) = kill_servers() {
error!("Failed to kill servers with error: {e}");
}
// clean up
info!("Cleaning up test directories ...");
fs::remove_dir_all("server1").map_err(|e| {
HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}"))
})?;
fs::remove_dir_all("server2").map_err(|e| {
HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}"))
})?;
ret
}
/// Actually run the tests. This will run:
/// - The standard test suite
/// - The persistence test suite
fn run_test_inner() -> HarnessResult<()> {
// first create the TLS keys
info!("Creating TLS key+cert");
let (cert, pkey) = mk_ca_cert().expect("Failed to create cert");
let mut certfile = fs::File::create("cert.pem").expect("failed to create cert.pem");
certfile.write_all(&cert.to_pem().unwrap()).unwrap();
let mut pkeyfile = fs::File::create("key.pem").expect("failed to create key.pem");
pkeyfile
.write_all(&pkey.private_key_to_pem_pkcs8().unwrap())
.unwrap();
fs::create_dir_all("server1").map_err(|e| {
HarnessError::Other(format!("Failed to create `server1` dir with error: {e}"))
})?;
fs::create_dir_all("server2").map_err(|e| {
HarnessError::Other(format!("Failed to create `server2` dir with error: {e}"))
})?;
// assemble commands
let target_folder = util::get_target_folder(BuildMode::Debug);
let standard_test_suite;
let persist_test_suite;
let build_cmd;
match util::get_var(util::VAR_TARGET) {
Some(target) => {
standard_test_suite = cmd!("cargo", "test", "--target", &target);
persist_test_suite = cmd!(
"cargo",
"test",
"--target",
&target,
"--features",
"persist-suite"
);
build_cmd = cmd!("cargo", "build", "-p", "skyd", "--target", &target);
}
None => {
standard_test_suite = cmd!("cargo", "test");
persist_test_suite = cmd!("cargo", "test", "--features", "persist-suite");
build_cmd = cmd!("cargo", "build", "-p", "skyd");
}
}
// build skyd
info!("Building server binary ...");
util::handle_child("build skyd", build_cmd)?;
// run standard test suite
run_with_servers(&target_folder, move || {
info!("Running standard test suite ...");
util::handle_child("standard test suite", standard_test_suite)?;
Ok(())
})?;
// run persistence tests
run_with_servers(&target_folder, move || {
info!("Running persistence test suite ...");
util::handle_child("standard test suite", persist_test_suite)?;
Ok(())
})?;
Ok(())
}
/// Generate certificates
fn mk_ca_cert() -> Result<(X509, PKey<Private>), ErrorStack> {
let rsa = Rsa::generate(2048)?;
let key_pair = PKey::from_rsa(rsa)?;
let mut x509_name = X509NameBuilder::new()?;
x509_name.append_entry_by_text("C", "US")?;
x509_name.append_entry_by_text("ST", "CA")?;
x509_name.append_entry_by_text("O", "Skytable")?;
x509_name.append_entry_by_text("CN", "sky-harness")?;
let x509_name = x509_name.build();
let mut cert_builder = X509::builder()?;
cert_builder.set_version(2)?;
let serial_number = {
let mut serial = BigNum::new()?;
serial.rand(159, MsbOption::MAYBE_ZERO, false)?;
serial.to_asn1_integer()?
};
cert_builder.set_serial_number(&serial_number)?;
cert_builder.set_subject_name(&x509_name)?;
cert_builder.set_issuer_name(&x509_name)?;
cert_builder.set_pubkey(&key_pair)?;
let not_before = Asn1Time::days_from_now(0)?;
cert_builder.set_not_before(&not_before)?;
let not_after = Asn1Time::days_from_now(365)?;
cert_builder.set_not_after(&not_after)?;
cert_builder.append_extension(BasicConstraints::new().critical().ca().build()?)?;
cert_builder.append_extension(
KeyUsage::new()
.critical()
.key_cert_sign()
.crl_sign()
.build()?,
)?;
let subject_key_identifier =
SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?;
cert_builder.append_extension(subject_key_identifier)?;
cert_builder.sign(&key_pair, MessageDigest::sha256())?;
let cert = cert_builder.build();
Ok((cert, key_pair))
}

@ -27,6 +27,7 @@
use crate::build::BuildMode;
use crate::{HarnessError, HarnessResult};
use std::env;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output};
pub type ExitCode = Option<i32>;
@ -56,6 +57,13 @@ pub fn get_child(desc: impl ToString, mut input: Command) -> HarnessResult<Child
}
}
pub fn assemble_command_from_slice<T: AsRef<OsStr>>(commands: impl AsRef<[T]>) -> Command {
let mut commands = commands.as_ref().iter();
let mut c = Command::new(commands.next().unwrap());
c.args(commands);
c
}
fn check_child_err(desc: impl ToString, output: Output) -> HarnessResult<()> {
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);

@ -35,7 +35,7 @@ pub mod terminal {
if stdout.set_color(ColorSpec::new().set_fg(color)).is_err() {
return Err(fmt::Error);
}
if write!(&mut stdout, "{}", item).is_err() {
if writeln!(&mut stdout, "{}", item).is_err() {
return Err(fmt::Error);
}
if stdout.reset().is_err() {

@ -25,8 +25,7 @@
*/
use super::{BGSave, Configset, PortConfig, SnapshotConfig, SnapshotPref, SslOpts, DEFAULT_IPV4};
pub(super) use libsky::TResult;
use crate::ROOT_DIR;
use std::fs;
// server tests
@ -336,15 +335,9 @@ fn tls_settings_fail_with_missing_required_values() {
}
/// Gets a `toml` file from `WORKSPACEROOT/examples/config-files`
fn get_toml_from_examples_dir(filename: &str) -> TResult<String> {
use std::path;
let curdir = path::Path::new(env!("CARGO_MANIFEST_DIR"));
let workspaceroot = curdir.ancestors().nth(1).unwrap();
let mut fileloc = path::PathBuf::from(workspaceroot);
fileloc.push("examples");
fileloc.push("config-files");
fileloc.push(filename);
Ok(fs::read_to_string(fileloc)?)
fn get_toml_from_examples_dir(filename: &str) -> String {
let path = format!("{ROOT_DIR}examples/config-files/{filename}");
fs::read_to_string(path).unwrap()
}
mod cfg_file_tests {
@ -364,7 +357,7 @@ mod cfg_file_tests {
#[test]
fn config_file_okay() {
let file = get_toml_from_examples_dir("template.toml").unwrap();
let file = get_toml_from_examples_dir("template.toml");
let toml = toml::from_str(&file).unwrap();
let cfg_from_file = cfgfile::from_file(toml);
assert!(cfg_from_file.is_mutated());
@ -389,14 +382,14 @@ mod cfg_file_tests {
#[test]
fn test_config_file_ok() {
let file = get_toml_from_examples_dir("skyd.toml").unwrap();
let file = get_toml_from_examples_dir("skyd.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(cfg.cfg, ConfigurationSet::default());
}
#[test]
fn test_config_file_noart() {
let file = get_toml_from_examples_dir("secure-noart.toml").unwrap();
let file = get_toml_from_examples_dir("secure-noart.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -414,7 +407,7 @@ mod cfg_file_tests {
#[test]
fn test_config_file_ipv6() {
let file = get_toml_from_examples_dir("ipv6.toml").unwrap();
let file = get_toml_from_examples_dir("ipv6.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -435,7 +428,7 @@ mod cfg_file_tests {
#[test]
fn test_config_file_template() {
let file = get_toml_from_examples_dir("template.toml").unwrap();
let file = get_toml_from_examples_dir("template.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -461,14 +454,14 @@ mod cfg_file_tests {
#[test]
fn test_config_file_bad_bgsave_section() {
let file = get_toml_from_examples_dir("badcfg2.toml").unwrap();
let file = get_toml_from_examples_dir("badcfg2.toml");
let cfg = cfgset_from_toml_str(file);
assert!(cfg.is_err());
}
#[test]
fn test_config_file_custom_bgsave() {
let file = get_toml_from_examples_dir("withcustombgsave.toml").unwrap();
let file = get_toml_from_examples_dir("withcustombgsave.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -490,7 +483,7 @@ mod cfg_file_tests {
* This test demonstrates a case where the user just said that BGSAVE is enabled.
* In that case, we will default to the 120 second duration
*/
let file = get_toml_from_examples_dir("bgsave-justenabled.toml").unwrap();
let file = get_toml_from_examples_dir("bgsave-justenabled.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -512,7 +505,7 @@ mod cfg_file_tests {
* This test demonstrates a case where the user just gave the value for every
* In that case, it means BGSAVE is enabled and set to `every` seconds
*/
let file = get_toml_from_examples_dir("bgsave-justevery.toml").unwrap();
let file = get_toml_from_examples_dir("bgsave-justevery.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,
@ -530,7 +523,7 @@ mod cfg_file_tests {
#[test]
fn test_config_file_snapshot() {
let file = get_toml_from_examples_dir("snapshot.toml").unwrap();
let file = get_toml_from_examples_dir("snapshot.toml");
let cfg = cfgset_from_toml_str(file).unwrap();
assert_eq!(
cfg.cfg,

Loading…
Cancel
Save