From b018f14e1cae055e93ac8825e86f04f9ee2612e8 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Thu, 17 Mar 2022 05:56:18 -0700 Subject: [PATCH] Add harness for test and release --- Cargo.lock | 102 +++++++++++++++++++++++++++++++ Cargo.toml | 1 + Makefile | 1 + harness/Cargo.toml | 12 ++++ harness/src/bundle.rs | 135 ++++++++++++++++++++++++++++++++++++++++++ harness/src/cli.rs | 75 +++++++++++++++++++++++ harness/src/error.rs | 51 ++++++++++++++++ harness/src/main.rs | 59 ++++++++++++++++++ harness/src/test.rs | 129 ++++++++++++++++++++++++++++++++++++++++ harness/src/util.rs | 84 ++++++++++++++++++++++++++ 10 files changed, 649 insertions(+) create mode 100644 harness/Cargo.toml create mode 100644 harness/src/bundle.rs create mode 100644 harness/src/cli.rs create mode 100644 harness/src/error.rs create mode 100644 harness/src/main.rs create mode 100644 harness/src/test.rs create mode 100644 harness/src/util.rs diff --git a/Cargo.lock b/Cargo.lock index 04e8fde6..07f14462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -115,6 +121,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.73" @@ -183,6 +210,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -356,6 +392,18 @@ dependencies = [ "windows-sys 0.30.0", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "foreign-types" version = "0.3.2" @@ -433,6 +481,16 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "harness" +version = "0.1.0" +dependencies = [ + "env_logger", + "libsky", + "log", + "zip", +] + [[package]] name = "hashbrown" version = "0.12.0" @@ -577,6 +635,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" @@ -1243,6 +1311,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.1.44" @@ -1488,3 +1576,17 @@ name = "yaml-rust" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time", +] diff --git a/Cargo.toml b/Cargo.toml index 6e892dc5..06d6a106 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "libstress", "stress-test", "sky-migrate", + "harness", ] [profile.release] diff --git a/Makefile b/Makefile index 56663624..26fc2798 100644 --- a/Makefile +++ b/Makefile @@ -178,3 +178,4 @@ deb: release-bundle checkcmd: @echo $(START_SERVER) @echo $(START_SERVER2) + @echo $(TARGET_FOLDER) \ No newline at end of file diff --git a/harness/Cargo.toml b/harness/Cargo.toml new file mode 100644 index 00000000..f06ff797 --- /dev/null +++ b/harness/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "harness" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libsky = { path = "../libsky" } +env_logger = "0.9.0" +log = "0.4.14" +zip = { version = "0.5.13", features = ["deflate"] } diff --git a/harness/src/bundle.rs b/harness/src/bundle.rs new file mode 100644 index 00000000..2683f7a1 --- /dev/null +++ b/harness/src/bundle.rs @@ -0,0 +1,135 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::{util, HarnessError, HarnessResult}; +use libsky::VERSION; +use std::fs; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; +use zip::{write::FileOptions, ZipWriter}; + +const BINARIES: [&str; 4] = ["skyd", "sky-bench", "skysh", "sky-migrate"]; + +fn concat_path(binary_name: &str, body: impl AsRef) -> PathBuf { + let mut pb = PathBuf::from(body.as_ref()); + #[cfg(windows)] + let binary_name = format!("{}.exe", binary_name); + pb.push(binary_name); + pb +} + +fn get_files_index(target_folder: &PathBuf) -> Vec { + let mut paths = Vec::with_capacity(3); + for binary in BINARIES { + paths.push(concat_path(binary, target_folder)); + } + paths +} + +fn get_bundle_name() -> String { + let mut filename = format!("sky-bundle-v{VERSION}"); + match util::get_var(util::VAR_ARTIFACT) { + Some(artifact) => { + filename.push('-'); + filename.push_str(&artifact); + } + None => {} + } + filename.push_str(".zip"); + filename +} + +pub fn run_bundle() -> HarnessResult<()> { + let mut build_args = vec!["build".to_owned()]; + let mut target_folder = PathBuf::from("target"); + match util::get_var(util::VAR_TARGET) { + Some(t) => { + build_args.push("--target".to_owned()); + build_args.push(t.to_string()); + target_folder.push(&t); + } + None => {} + }; + target_folder.push("release"); + + // assemble build args + build_args.extend([ + "-p".into(), + "skyd".into(), + "-p".into(), + "sky-bench".into(), + "-p".into(), + "skysh".into(), + "-p".into(), + "sky-migrate".into(), + "--release".into(), + ]); + let mut cmd = Command::new("cargo"); + cmd.args(&build_args); + util::handle_child("build release binaries", cmd)?; + + // now package + package_binaries(target_folder)?; + Ok(()) +} + +fn package_binaries(target_folder: PathBuf) -> HarnessResult<()> { + // get the file index + let file_index = get_files_index(&target_folder); + // get the bundle file name + let bundle_file_name = get_bundle_name(); + // create the bundle file + let bundle_file = fs::File::create(&bundle_file_name) + .map_err(|e| HarnessError::Other(format!("Failed to create ZIP file with error: {e}")))?; + // init zip writer + let mut zip = ZipWriter::new(bundle_file); + // create a temp buffer + let mut buffer = Vec::new(); + // ZIP settings + let options = FileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .unix_permissions(0o755); + for file in file_index { + let path = file.as_path(); + let name = path.strip_prefix(Path::new(&target_folder)).unwrap(); + #[allow(deprecated)] + zip.start_file_from_path(name, options).unwrap(); + let mut f = fs::File::open(path).map_err(|e| { + HarnessError::Other(format!( + "Failed to add file `{}` to ZIP with error: {e}", + path.to_string_lossy() + )) + })?; + f.read_to_end(&mut buffer).unwrap(); + zip.write_all(&*buffer).unwrap(); + buffer.clear(); + } + zip.finish().unwrap(); + Ok(()) +} diff --git a/harness/src/cli.rs b/harness/src/cli.rs new file mode 100644 index 00000000..7e1a4006 --- /dev/null +++ b/harness/src/cli.rs @@ -0,0 +1,75 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::{HarnessError, HarnessResult}; +use std::{env, process}; + +const HELP: &str = "\ +harness +A harness for Skytable's test suite + +OPTIONS: + harness [SUBCOMMAND] + +SUBCOMMANDS: + test Run the full test suite + bundle Build the bundle\ +"; + +pub enum HarnessWhat { + Test, + Bundle, +} + +impl HarnessWhat { + const CLI_TEST: &'static str = "test"; + const CLI_BUNDLE: &'static str = "bundle"; + const CLI_ARG_HELP: &'static str = "--help"; + const CLI_ARG_HELP_SHORT: &'static str = "-h"; + pub fn from_env() -> HarnessResult { + let args: Vec = env::args().skip(1).collect(); + if args.is_empty() { + display_help(); + } else if args.len() != 1 { + return Err(HarnessError::BadArguments(format!( + "expected one argument. found {} args", + args.len() + ))); + } + let ret = match args[0].as_str() { + Self::CLI_TEST => HarnessWhat::Test, + Self::CLI_BUNDLE => HarnessWhat::Bundle, + Self::CLI_ARG_HELP_SHORT | Self::CLI_ARG_HELP => display_help(), + unknown_arg => return Err(HarnessError::UnknownCommand(unknown_arg.to_string())), + }; + Ok(ret) + } +} + +fn display_help() -> ! { + println!("{}", HELP); + process::exit(0x00) +} diff --git a/harness/src/error.rs b/harness/src/error.rs new file mode 100644 index 00000000..38f2c806 --- /dev/null +++ b/harness/src/error.rs @@ -0,0 +1,51 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::util::ExitCode; +use std::fmt; + +pub type HarnessResult = Result; +#[derive(Debug)] +pub enum HarnessError { + UnknownCommand(String), + BadArguments(String), + ChildError(&'static str, ExitCode), + Other(String), +} + +impl fmt::Display for HarnessError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + HarnessError::BadArguments(arg) => write!(f, "Bad arguments: `{}`", arg), + HarnessError::UnknownCommand(cmd) => write!(f, "Unknown command: `{}`", cmd), + HarnessError::ChildError(desc, code) => match code { + Some(code) => write!(f, "The child (`{desc}`) exited with code {code}"), + None => write!(f, "The child (`{desc}`) exited with a non-zero code"), + }, + HarnessError::Other(other) => write!(f, "{other}"), + } + } +} diff --git a/harness/src/main.rs b/harness/src/main.rs new file mode 100644 index 00000000..592c82eb --- /dev/null +++ b/harness/src/main.rs @@ -0,0 +1,59 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[macro_use] +extern crate log; +#[macro_use] +mod util; +mod bundle; +mod cli; +mod error; +mod test; +use crate::{ + cli::HarnessWhat, + error::{HarnessError, HarnessResult}, +}; +use env_logger::Builder; +use std::{env, process}; + +fn main() { + Builder::new() + .parse_filters(&env::var("SKYHARNESS_LOG").unwrap_or_else(|_| "info".to_owned())) + .init(); + if let Err(e) = runner() { + eprintln!("harness failed with: {}", e); + process::exit(0x01); + } +} + +fn runner() -> HarnessResult<()> { + let harness = cli::HarnessWhat::from_env()?; + match harness { + HarnessWhat::Test => test::run_test()?, + HarnessWhat::Bundle => bundle::run_bundle()?, + } + Ok(()) +} diff --git a/harness/src/test.rs b/harness/src/test.rs new file mode 100644 index 00000000..82ee05d4 --- /dev/null +++ b/harness/src/test.rs @@ -0,0 +1,129 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::{util, HarnessError, HarnessResult}; +use std::fs; +use std::process::Child; +use std::process::Command; + +const WORKSPACE_ROOT: &str = env!("ROOT_DIR"); + +pub fn get_run_server_cmd(server_id: &'static str, cmd_payload: &[String]) -> Command { + let mut cmd = Command::new("cargo"); + cmd.args(cmd_payload); + cmd.arg("--"); + cmd.arg("--withconfig"); + cmd.arg(format!("{WORKSPACE_ROOT}ci/{server_id}.toml")); + cmd.current_dir(server_id); + cmd +} + +pub fn start_servers(s1_cmd: Command, s2_cmd: Command) -> HarnessResult<(Child, Child)> { + info!("Starting server1 ..."); + let s1 = util::get_child("start server1", s1_cmd)?; + util::sleep_sec(10); + info!("Starting server2 ..."); + let s2 = util::get_child("start server2", s2_cmd)?; + util::sleep_sec(10); + Ok((s1, s2)) +} + +fn kill_servers() -> HarnessResult<()> { + util::handle_child("kill servers", cmd!("pkill", "skyd"))?; + // sleep + util::sleep_sec(10); + Ok(()) +} + +pub fn run_test() -> HarnessResult<()> { + let ret = run_test_inner(); + kill_servers()?; + + // clean up + fs::remove_dir_all("server1").map_err(|e| { + HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}")) + })?; + fs::remove_dir_all("server2").map_err(|e| { + HarnessError::Other(format!("Failed to remove dir `server1` with error: {e}")) + })?; + ret +} + +pub fn run_test_inner() -> HarnessResult<()> { + // first create the TLS keys + info!("Creating TLS key+cert"); + util::handle_child("generate TLS key+cert", cmd!("bash", "ci/ssl.sh"))?; + util::handle_child( + "create server1 directory", + cmd!("mkdir", "-p", "server1", "server2"), + )?; + + // assemble commands + let mut cmd: Vec = vec!["run".to_string(), "-p".to_string(), "skyd".to_string()]; + let standard_test_suite; + let persist_test_suite; + let build_cmd; + match util::get_var(util::VAR_TARGET) { + Some(target) => { + cmd.push("--target".into()); + cmd.push(target.to_string()); + standard_test_suite = cmd!("cargo", "test", "--target", &target); + persist_test_suite = cmd!( + "cargo", + "test", + "--target", + &target, + "--features", + "persist-suite" + ); + build_cmd = cmd!("cargo", "build", "-p", "skyd", "--target", &target); + } + None => { + standard_test_suite = cmd!("cargo", "test"); + persist_test_suite = cmd!("cargo", "test", "--features", "persist-suite"); + build_cmd = cmd!("cargo", "build", "-p", "skyd"); + } + } + + // build skyd + util::handle_child("build skyd", build_cmd)?; + let s1_cmd = get_run_server_cmd("server1", &cmd); + let s2_cmd = get_run_server_cmd("server2", &cmd); + + // start the servers, run tests and kill + let (_s1, _s2) = start_servers(s1_cmd, s2_cmd)?; + info!("All servers started. Now running standard test suite ..."); + util::handle_child("standard test suite", standard_test_suite)?; + kill_servers()?; + + // start server up again, run tests and kill + let s1_cmd = get_run_server_cmd("server1", &cmd); + let s2_cmd = get_run_server_cmd("server2", &cmd); + let (_s1, _s2) = start_servers(s1_cmd, s2_cmd)?; + util::handle_child("standard test suite", persist_test_suite)?; + + Ok(()) +} diff --git a/harness/src/util.rs b/harness/src/util.rs new file mode 100644 index 00000000..61e968fd --- /dev/null +++ b/harness/src/util.rs @@ -0,0 +1,84 @@ +/* + * Created on Thu Mar 17 2022 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2022, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::process::ExitStatus; +use crate::{HarnessError, HarnessResult}; +use std::env; +use std::io::Result as IoResult; +use std::process::Child; +use std::process::Command; +pub type ExitCode = Option; + +pub const VAR_TARGET: &str = "TARGET"; +pub const VAR_ARTIFACT: &str = "ARTIFACT"; + +pub fn get_var(var: &str) -> Option { + env::var_os(var).map(|v| v.to_string_lossy().to_string()) +} + +pub fn handle_exitstatus(desc: &'static str, status: IoResult) -> HarnessResult<()> { + match status { + Ok(status) => { + if status.success() { + Ok(()) + } else { + Err(HarnessError::ChildError(desc, status.code())) + } + } + Err(e) => Err(HarnessError::Other(format!( + "Failed to get exitcode while running `{desc}`. this error happened: {e}" + ))), + } +} + +pub fn get_child(desc: impl ToString, mut input: Command) -> HarnessResult { + let desc = desc.to_string(); + match input.spawn() { + Ok(child) => Ok(child), + Err(e) => Err(HarnessError::Other(format!( + "Failed to spawn process for `{desc}` with error: {e}" + ))), + } +} + +pub fn handle_child(desc: &'static str, input: Command) -> HarnessResult<()> { + self::handle_exitstatus(desc, self::get_child(desc, input)?.wait()) +} + +pub fn sleep_sec(secs: u64) { + std::thread::sleep(std::time::Duration::from_secs(secs)) +} + +#[macro_export] +macro_rules! cmd { + ($base:expr, $($cmd:expr),*) => {{ + let mut cmd = ::std::process::Command::new($base); + $( + cmd.arg($cmd); + )* + cmd + }}; +}