diff --git a/.cargo/config.toml b/.cargo/config.toml index ac33e0c8..7c9f7c37 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,2 @@ [env] ROOT_DIR = { value = "", relative = true } -TEST_ORIGIN_KEY = "4527387f92a381cbe804593f33991d327d456a97" diff --git a/.gitignore b/.gitignore index 79cf9637..a40afb49 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ snapstore.partmap .devcontainer *.deb .skytest_* -*.pem \ No newline at end of file +*.pem +passphrase.txt \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 48518883..e72855d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,37 +37,17 @@ dependencies = [ "memchr", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", "syn 2.0.28", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -114,15 +94,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -214,38 +185,21 @@ dependencies = [ ] [[package]] -name = "clap" -version = "2.34.0" +name = "constant_time_eq" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim", - "textwrap", - "unicode-width", - "vec_map", - "yaml-rust", -] +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] -name = "clipboard-win" -version = "4.5.0" +name = "core-foundation" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ - "error-code", - "str-buf", - "winapi", + "core-foundation-sys", + "libc", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -328,31 +282,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -369,12 +298,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" -[[package]] -name = "devtimer" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1" - [[package]] name = "digest" version = "0.10.7" @@ -392,12 +315,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -[[package]] -name = "endian-type" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" - [[package]] name = "env_logger" version = "0.10.0" @@ -439,25 +356,10 @@ dependencies = [ ] [[package]] -name = "error-code" -version = "2.3.1" +name = "fastrand" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" -dependencies = [ - "libc", - "str-buf", -] - -[[package]] -name = "fd-lock" -version = "3.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" -dependencies = [ - "cfg-if", - "rustix", - "windows-sys", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" @@ -555,7 +457,7 @@ dependencies = [ "log", "openssl", "powershell_script", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", + "skytable", "zip", ] @@ -565,15 +467,6 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.2" @@ -589,15 +482,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys", -] - [[package]] name = "humantime" version = "2.1.0" @@ -629,7 +513,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi", "rustix", "windows-sys", ] @@ -677,9 +561,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libsky" @@ -743,44 +627,31 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", - "log", "wasi", "windows-sys", ] [[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec", -] - -[[package]] -name = "nix" -version = "0.26.2" +name = "native-tls" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ - "bitflags 1.3.2", - "cfg-if", + "lazy_static", "libc", - "static_assertions", -] - -[[package]] -name = "ntapi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" -dependencies = [ - "winapi", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] @@ -789,7 +660,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi", "libc", ] @@ -834,6 +705,12 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "openssl-src" version = "111.27.0+1.1.1v" @@ -961,16 +838,6 @@ dependencies = [ "scheduled-thread-pool", ] -[[package]] -name = "radix_trie" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" -dependencies = [ - "endian-type", - "nibble_vec", -] - [[package]] name = "rand" version = "0.8.5" @@ -1091,35 +958,21 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustyline" -version = "12.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" -dependencies = [ - "bitflags 2.3.3", - "cfg-if", - "clipboard-win", - "fd-lock", - "home", - "libc", - "log", - "memchr", - "nix", - "radix_trie", - "scopeguard", - "unicode-segmentation", - "unicode-width", - "utf8parse", - "winapi", -] - [[package]] name = "ryu" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + [[package]] name = "scheduled-thread-pool" version = "0.2.7" @@ -1135,6 +988,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.183" @@ -1155,17 +1031,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "serde_json" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_yaml" version = "0.9.25" @@ -1201,27 +1066,6 @@ dependencies = [ "digest", ] -[[package]] -name = "signal-hook" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1235,26 +1079,8 @@ dependencies = [ name = "sky-bench" version = "0.8.0" dependencies = [ - "clap", - "devtimer", - "env_logger", "libstress", - "log", - "rand", - "serde", - "serde_json", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", -] - -[[package]] -name = "sky-migrate" -version = "0.8.0" -dependencies = [ - "bincode", - "clap", - "env_logger", - "log", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)", + "skytable", ] [[package]] @@ -1297,35 +1123,22 @@ dependencies = [ name = "skysh" version = "0.8.0" dependencies = [ - "clap", - "crossterm", - "lazy_static", "libsky", - "rustyline", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)", - "tokio", + "skytable", ] [[package]] name = "skytable" version = "0.8.0" -source = "git+https://github.com/skytable/client-rust?branch=next#a55fbdc964e34c75c99404ea2395d03fd302daee" +source = "git+https://github.com/skytable/client-rust.git?branch=octave#b2b0ea7197d9a3425809ce269e30b74ddd3eb340" dependencies = [ "async-trait", "bb8", - "bytes", - "openssl", + "native-tls", "r2d2", + "rand", "tokio", - "tokio-openssl", -] - -[[package]] -name = "skytable" -version = "0.8.0" -source = "git+https://github.com/skytable/client-rust.git#a55fbdc964e34c75c99404ea2395d03fd302daee" -dependencies = [ - "r2d2", + "tokio-native-tls", ] [[package]] @@ -1345,46 +1158,14 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "str-buf" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" - -[[package]] -name = "stress-test" -version = "0.1.0" -dependencies = [ - "crossbeam-channel", - "devtimer", - "env_logger", - "libstress", - "log", - "rand", - "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)", - "sysinfo", -] - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "subtle" version = "2.5.0" @@ -1414,18 +1195,16 @@ dependencies = [ ] [[package]] -name = "sysinfo" -version = "0.29.7" +name = "tempfile" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", - "core-foundation-sys", - "libc", - "ntapi", - "once_cell", - "rayon", - "winapi", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", ] [[package]] @@ -1437,15 +1216,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "time" version = "0.3.25" @@ -1465,9 +1235,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "tokio" -version = "1.30.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1484,15 +1254,25 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", "syn 2.0.28", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-openssl" version = "0.6.3" @@ -1517,30 +1297,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - [[package]] name = "unsafe-libyaml" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - [[package]] name = "uuid" version = "1.4.1" @@ -1569,12 +1331,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" @@ -1684,12 +1440,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" -[[package]] -name = "yaml-rust" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" - [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index 448c2321..a07132a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,6 @@ members = [ "sky-bench", "sky-macros", "libstress", - "stress-test", - "sky-migrate", "harness", ] diff --git a/ci/server1.toml b/ci/server1.toml deleted file mode 100644 index dad19e6c..00000000 --- a/ci/server1.toml +++ /dev/null @@ -1,9 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 -noart = true - -[ssl] -key="../key.pem" -chain="../cert.pem" -port = 2004 diff --git a/ci/server1.yaml b/ci/server1.yaml new file mode 100644 index 00000000..fc3aaa18 --- /dev/null +++ b/ci/server1.yaml @@ -0,0 +1,17 @@ +system: + mode: prod + +auth: + plugin: pwd + root_pass: password12345678 + +endpoints: + insecure: + host: 127.0.0.1 + port: 2003 + secure: + host: 127.0.0.1 + port: 2004 + cert: ../cert.pem + private_key: ../key.pem + pkey_passphrase: ../passphrase.txt diff --git a/ci/server2.toml b/ci/server2.toml deleted file mode 100644 index 05083faa..00000000 --- a/ci/server2.toml +++ /dev/null @@ -1,12 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2005 -noart = true - -[auth] -origin_key = "4527387f92a381cbe804593f33991d327d456a97" - -[ssl] -key = "../key.pem" -chain = "../cert.pem" -port = 2006 diff --git a/ci/server3.toml b/ci/server3.toml deleted file mode 100644 index 69d6b24b..00000000 --- a/ci/server3.toml +++ /dev/null @@ -1,14 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2007 -noart = true - -[snapshot] -every = 3600 -atmost = 4 -failsafe = true - -[ssl] -key = "../key.pem" -chain = "../cert.pem" -port = 2008 diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0402ce65..68312688 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -9,13 +9,4 @@ edition = "2021" [dependencies] # internal deps libsky = { path = "../libsky" } -skytable = { git = "https://github.com/skytable/client-rust", branch = "next", features = [ - "aio", - "aio-sslv", -], default-features = false } -# external deps -tokio = { version = "1.29.1", features = ["full"] } -clap = { version = "2", features = ["yaml"] } -rustyline = "12.0.0" -crossterm = "0.26.1" -lazy_static = "1.4.0" +skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" } diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs deleted file mode 100644 index 12ad8e80..00000000 --- a/cli/src/argparse.rs +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Created on Wed Jul 01 2020 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{runner::Runner, tokenizer}, - clap::{load_yaml, App}, - crossterm::{ - cursor, execute, - terminal::{Clear, ClearType}, - }, - libsky::{URL, VERSION}, - rustyline::{config::Configurer, error::ReadlineError, DefaultEditor}, - skytable::{Pipeline, Query}, - std::{io::stdout, process}, -}; - -const ADDR: &str = "127.0.0.1"; -const SKYSH_HISTORY_FILE: &str = ".sky_history"; - -const HELP_TEXT: &str = r#" -███████ ██  ██ ██  ██ ████████  █████  ██████  ██  ███████ -██      ██  ██   ██  ██     ██    ██   ██ ██   ██ ██  ██ -███████ █████    ████   ██  ███████ ██████  ██  █████ -     ██ ██  ██   ██   ██  ██   ██ ██   ██ ██  ██ -███████ ██  ██  ██  ██  ██  ██ ██████  ███████ ███████ - -Welcome to Skytable's interactive shell (REPL) environment. Using the Skytable -shell, you can create, read, update or delete data on your remote Skytable -instance. When you connect to your database instance, you'll be connected to -the `default` table in the `default` keyspace. This table has binary keys and -binary values as the default data type. Here's a brief guide on doing some -everyday tasks: - -(1) Running actions -================================================================================ -An action is like a shell command: it starts with a name and contains arguments! -To run actions, simply type them out, like "set x 100" or "inspect table mytbl" -and hit enter. - -(2) Running shell commands -================================================================================ -Shell commands are those which are provided by `skysh` and are not supported by -the server. These enable you to do convenient things like: -- "exit": exits the shell -- "clear": clears the terminal screen - -Apart from these, you can use the following shell commands: -- "!pipe": Lets you create a pipeline. Terminate with a semicolon (`;`) -- "!help": Brings up this help menu -- "?": Describes what the built-in shell command is for - -With Skytable in your hands, the sky is the only limit on what you can create!"#; - -const SKY_WELCOME: &str = " -Welcome to Skytable's interactive shell (REPL) environment. For usage and help -within the shell, you can run `!help` anytime. Now that you have Skytable in -your hands, the sky is the only limit on what you can create! -"; - -/// This creates a REPL on the command line and also parses command-line arguments -/// -/// Anything that is entered following a return, is parsed into a query and is -/// written to the socket (which is either `localhost:2003` or it is determined by -/// command line parameters) -pub async fn start_repl() { - let mut skysh_blank: String = " > ".to_owned(); - let mut skysh_prompt: String = "skysh@default:default> ".to_owned(); - let mut did_swap = false; - - macro_rules! readln { - ($editor:expr) => { - match $editor.readline(&skysh_blank) { - Ok(l) => l, - Err(ReadlineError::Interrupted | ReadlineError::Eof) => return, - Err(err) => fatal!("ERROR: Failed to read line with error: {}", err), - } - }; - } - - let cfg_layout = load_yaml!("./cli.yml"); - let matches = App::from_yaml(cfg_layout).get_matches(); - let host = libsky::option_unwrap_or!(matches.value_of("host"), ADDR); - let port = match matches.value_of("port") { - Some(p) => match p.parse::() { - Ok(p) => p, - Err(_) => fatal!("Invalid port"), - }, - None => 2003, - }; - let mut editor = match DefaultEditor::new() { - Ok(e) => e, - Err(e) => fatal!("Editor init error: {}", e), - }; - editor.set_auto_add_history(true); - editor - .set_history_ignore_dups(true) - .unwrap_or_else(|_| fatal!("couldn't set up terminal")); - editor.bind_sequence( - rustyline::KeyEvent( - rustyline::KeyCode::BracketedPasteStart, - rustyline::Modifiers::NONE, - ), - rustyline::Cmd::Noop, - ); - let con = match matches.value_of("cert") { - Some(cert) => Runner::new_secure(host, port, cert).await, - None => Runner::new_insecure(host, port).await, - }; - let mut runner = match con { - Ok(c) => c, - Err(e) => fatal!("Failed to connect to server with error: {}", e), - }; - - macro_rules! checkswap { - () => { - if did_swap { - // noice, we need to poll for the location of the new entity - runner - .check_entity(&mut skysh_blank, &mut skysh_prompt) - .await; - } - }; - } - - if let Some(eval_expr) = matches.values_of("eval") { - for eval_expr in eval_expr { - if !eval_expr.is_empty() { - runner.run_query(eval_expr).await; - } - } - process::exit(0x00); - } - println!("Skytable v{} | {}", VERSION, URL); - match editor.load_history(SKYSH_HISTORY_FILE) { - Ok(_) => {} - Err(e) => match e { - ReadlineError::Io(e) if e.kind() == std::io::ErrorKind::NotFound => { - println!("{}", SKY_WELCOME) - } - _ => fatal!("Failed to read history file with error: {}", e), - }, - } - loop { - match editor.readline(&skysh_prompt) { - Ok(mut line) => { - macro_rules! tokenize { - ($inp:expr) => { - match tokenizer::get_query($inp) { - Ok(q) => q, - Err(e) => { - eskysh!(e); - continue; - } - } - }; - () => { - tokenize!(line.as_bytes()) - }; - } - match line.to_lowercase().as_str() { - "exit" => break, - "clear" => { - clear_screen(); - continue; - } - "help" => { - println!("To get help, run `!help`"); - continue; - } - _ => { - if line.is_empty() { - continue; - } - match line.as_bytes()[0] { - b'#' => continue, - b'!' => { - match &line.as_bytes()[1..] { - b"" => eskysh!("Bad shell command"), - b"help" => println!("{}", HELP_TEXT), - b"pipe" => { - // so we need to handle a pipeline - let mut pipeline = Pipeline::new(); - line = readln!(editor); - loop { - did_swap = line - .get(..3) - .map(|v| v.eq_ignore_ascii_case("use")) - .unwrap_or(did_swap); - if !line.is_empty() { - if *(line.as_bytes().last().unwrap()) == b';' { - break; - } else { - let q: Query = tokenize!(); - pipeline.push(q); - } - } - line = readln!(editor); - } - if line.len() > 1 { - line.drain(line.len() - 1..); - let q: Query = tokenize!(); - pipeline.push(q); - } - runner.run_pipeline(pipeline).await; - checkswap!(); - } - _ => eskysh!("Unknown shell command"), - } - continue; - } - b'?' => { - // handle explanation for a shell command - print_help(&line); - continue; - } - _ => {} - } - while line.len() >= 2 && line[line.len() - 2..].as_bytes().eq(br#" \"#) { - // continuation on next line - let cl = readln!(editor); - line.drain(line.len() - 2..); - line.push_str(&cl); - } - did_swap = line - .get(..3) - .map(|v| v.eq_ignore_ascii_case("use")) - .unwrap_or(did_swap); - runner.run_query(&line).await; - checkswap!(); - } - } - } - Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, - Err(err) => fatal!("ERROR: Failed to read line with error: {}", err), - } - } - editor - .save_history(SKYSH_HISTORY_FILE) - .map_err(|e| { - fatal!("ERROR: Failed to save history with error: '{}'", e); - }) - .unwrap(); -} - -fn print_help(line: &str) { - match &line.as_bytes()[1..] { - b"" => eskysh!("Bad shell command"), - b"help" => println!("`!help` shows the help menu"), - b"exit" => println!("`exit` ends the shell session"), - b"clear" => println!("`clear` clears the terminal screen"), - b"pipe" | b"!pipe" => println!("`!pipe` lets you run pipelines using the shell"), - _ => eskysh!("Unknown shell command"), - } -} - -fn clear_screen() { - let mut stdout = stdout(); - execute!(stdout, Clear(ClearType::All)).expect("Failed to clear screen"); - execute!(stdout, cursor::MoveTo(0, 0)).expect("Failed to move cursor to origin"); -} diff --git a/cli/src/cli.yml b/cli/src/cli.yml deleted file mode 100644 index db562b18..00000000 --- a/cli/src/cli.yml +++ /dev/null @@ -1,55 +0,0 @@ -# -# Created on Tue Nov 03 2020 -# -# This file is a part of Skytable -# Copyright (c) 2020, Sayan Nandan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# - -name: Skytable Shell -version: 0.8.0 -author: Sayan N. -about: The Skytable Shell (skysh) -args: - - host: - short: h - required: false - long: host - value_name: host - help: Sets the remote host to connect to - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the remote port to connect to - takes_value: true - - eval: - short: e - required: false - long: eval - multiple: true - value_name: expression - help: Run an expression without REPL - takes_value: true - - cert: - short: C - required: false - long: sslcert - value_name: cert - help: Sets the PEM certificate to use for SSL connections - takes_value: true diff --git a/cli/src/macros.rs b/cli/src/macros.rs deleted file mode 100644 index cdc809d6..00000000 --- a/cli/src/macros.rs +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Created on Wed Nov 03 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -macro_rules! write_str { - ($st:ident) => { - println!("\"{}\"", $st) - }; - ($idx:ident, $st:ident) => { - println!("({}) \"{}\"", $idx, $st) - }; -} - -macro_rules! write_binstr { - ($st:ident) => { - println!("{}", BinaryData($st)) - }; - ($idx:ident, $st:ident) => { - println!("({}) {}", $idx, BinaryData($st)) - }; -} - -macro_rules! write_int { - ($int:ident) => { - println!("{}", $int) - }; - ($idx:ident, $st:ident) => { - println!("({}) \"{}\"", $idx, $st) - }; -} - -macro_rules! write_err { - ($idx:expr, $err:ident) => { - err!(if let Some(idx) = $idx { - format!("({}) ({})\n", idx, $err) - } else { - format!("({})\n", $err) - }) - }; - ($idx:ident, $err:literal) => { - err!( - (if let Some(idx) = $idx { - format!("({}) ({})\n", idx, $err) - } else { - format!("({})\n", $err) - }) - ) - }; -} - -macro_rules! write_okay { - () => { - crossterm::execute!( - std::io::stdout(), - SetForegroundColor(Color::Cyan), - Print("(Okay)\n".to_string()), - ResetColor - ) - .expect("Failed to write to stdout") - }; - ($idx:ident) => { - crossterm::execute!( - std::io::stdout(), - SetForegroundColor(Color::Cyan), - Print(format!("({}) (Okay)\n", $idx)), - ResetColor - ) - .expect("Failed to write to stdout") - }; -} - -macro_rules! err { - ($input:expr) => { - crossterm::execute!( - std::io::stdout(), - ::crossterm::style::SetForegroundColor(::crossterm::style::Color::Red), - ::crossterm::style::Print($input), - ::crossterm::style::ResetColor - ) - .expect("Failed to write to stdout") - }; -} - -macro_rules! eskysh { - ($e:expr) => { - err!(format!("[SKYSH ERROR] {}\n", $e)) - }; -} - -macro_rules! fatal { - ($e:expr) => {{ - err!($e); - ::std::process::exit(0x01); - }}; - ($e:expr, $desc:expr) => {{ - err!(format!($e, $desc)); - println!(); - ::std::process::exit(0x01) - }}; -} diff --git a/cli/src/main.rs b/cli/src/main.rs index fd90672e..57a21b8e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Jul 01 2020 + * Created on Wed Nov 15 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2020, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,20 +24,4 @@ * */ -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] - -#[macro_use] -mod macros; -mod argparse; -mod runner; -mod tokenizer; -// tests -#[cfg(test)] -mod tests; - -#[tokio::main] -async fn main() { - argparse::start_repl().await; - println!("Goodbye!"); -} +fn main() {} diff --git a/cli/src/runner.rs b/cli/src/runner.rs deleted file mode 100644 index f9ce45ac..00000000 --- a/cli/src/runner.rs +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Created on Wed May 12 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::tokenizer, - core::fmt, - crossterm::style::{Color, Print, ResetColor, SetForegroundColor}, - skytable::{ - aio, error::Error, types::Array, types::FlatElement, Element, Pipeline, Query, RespCode, - }, -}; - -type SkyResult = Result; - -pub enum Runner { - Insecure(aio::Connection), - Secure(aio::TlsConnection), -} - -impl Runner { - pub async fn new_insecure(host: &str, port: u16) -> SkyResult { - let con = aio::Connection::new(host, port).await?; - Ok(Self::Insecure(con)) - } - pub async fn new_secure(host: &str, port: u16, cert: &str) -> SkyResult { - let con = aio::TlsConnection::new(host, port, cert).await?; - Ok(Self::Secure(con)) - } - pub async fn run_pipeline(&mut self, pipeline: Pipeline) { - let ret = match self { - Self::Insecure(con) => con.run_pipeline(pipeline).await, - Self::Secure(con) => con.run_pipeline(pipeline).await, - }; - let retok = match ret { - Ok(r) => r, - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - }; - for (idx, resp) in retok - .into_iter() - .enumerate() - .map(|(idx, resp)| (idx + 1, resp)) - { - println!("[Response {}]", idx); - print_element(resp); - } - } - pub async fn run_query(&mut self, unescaped: &str) { - let query: Query = match tokenizer::get_query(unescaped.as_bytes()) { - Ok(q) => q, - Err(e) => { - err!(format!("[Syntax Error: {}]\n", e)); - return; - } - }; - let ret = match self { - Self::Insecure(con) => con.run_query_raw(&query).await, - Self::Secure(con) => con.run_query_raw(&query).await, - }; - match ret { - Ok(resp) => print_element(resp), - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - } - } - pub async fn check_entity(&mut self, blank: &mut String, prompt: &mut String) { - let query: Query = tokenizer::get_query(b"whereami").unwrap(); - let ret = match self { - Self::Insecure(con) => con.run_query_raw(&query).await, - Self::Secure(con) => con.run_query_raw(&query).await, - }; - let ret = match ret { - Ok(resp) => resp, - Err(e) => fatal!("An I/O error occurred while querying: {}", e), - }; - match ret { - Element::Array(Array::NonNullStr(srr)) => match srr.len() { - 1 => { - *blank = format!(" {blank}> ", blank = " ".repeat(srr[0].len())); - *prompt = format!("skysh@{ks}> ", ks = srr[0]); - } - 2 => { - let ks = &srr[0]; - let tbl = &srr[1]; - *blank = format!( - " {blank}> ", - blank = " ".repeat(ks.len() + tbl.len() + 1) - ); - *prompt = format!("skysh@{ks}:{tbl}> ", ks = ks, tbl = tbl); - } - count => fatal!( - "The server returned {} IDs while checking entity state", - count - ), - }, - _ => fatal!("The server returned the wrong data type for entity state check"), - } - } -} - -fn print_float(float: f32, idx: Option) { - if let Some(idx) = idx { - println!("({idx}) {float}") - } else { - println!("{float}"); - } -} - -fn print_element(el: Element) { - match el { - Element::String(st) => write_str!(st), - Element::Binstr(st) => write_binstr!(st), - Element::Array(Array::Bin(brr)) => print_bin_array(brr), - Element::Array(Array::Str(srr)) => print_str_array(srr), - Element::RespCode(r) => print_rcode(r, None), - Element::UnsignedInt(int) => write_int!(int), - Element::Array(Array::Flat(frr)) => write_flat_array(frr), - Element::Array(Array::Recursive(a)) => print_array(a), - Element::Array(Array::NonNullBin(nbrr)) => print_array_nonnull_bin(nbrr), - Element::Array(Array::NonNullStr(nsrr)) => print_array_nonnull_str(nsrr), - Element::Float(float) => print_float(float, None), - _ => eskysh!("The server possibly sent a newer data type that we can't parse"), - } -} - -fn print_rcode(rcode: RespCode, idx: Option) { - match rcode { - RespCode::Okay => write_okay!(), - RespCode::ActionError => write_err!(idx, "Action Error"), - RespCode::ErrorString(st) => write_err!(idx, st), - RespCode::OtherError => write_err!(idx, "Other Error"), - RespCode::NotFound => write_err!(idx, "Not Found"), - RespCode::OverwriteError => write_err!(idx, "Overwrite Error"), - RespCode::PacketError => write_err!(idx, "Packet Error"), - RespCode::ServerError => write_err!(idx, "Server Error"), - RespCode::UnknownDataType => write_err!(idx, "Unknown data type"), - RespCode::EncodingError => write_err!(idx, "Encoding error"), - RespCode::AuthBadCredentials => write_err!(idx, "auth bad credentials"), - RespCode::AuthPermissionError => write_err!(idx, "auth permission error"), - _ => write_err!(idx, "Unknown error"), - } -} - -fn print_bin_array(bin_array: Vec>>) { - bin_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - match elem { - Some(ele) => { - write_binstr!(idx, ele); - } - None => print_rcode(RespCode::NotFound, Some(idx)), - } - }) -} - -fn print_str_array(str_array: Vec>) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - match elem { - Some(ele) => { - write_str!(idx, ele); - } - None => print_rcode(RespCode::NotFound, Some(idx)), - } - }) -} - -fn print_array_nonnull_str(str_array: Vec) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - write_str!(idx, elem) - }) -} - -fn print_array_nonnull_bin(str_array: Vec>) { - str_array.into_iter().enumerate().for_each(|(idx, elem)| { - let idx = idx + 1; - write_binstr!(idx, elem) - }) -} - -fn write_flat_array(flat_array: Vec) { - for (idx, item) in flat_array.into_iter().enumerate() { - let idx = idx + 1; - match item { - FlatElement::String(st) => write_str!(idx, st), - FlatElement::Binstr(st) => { - write_binstr!(idx, st) - } - FlatElement::RespCode(rc) => print_rcode(rc, Some(idx)), - FlatElement::UnsignedInt(int) => write_int!(int, idx), - _ => eskysh!("Element typed cannot yet be parsed"), - } - } -} - -fn print_array(array: Vec) { - for (idx, item) in array.into_iter().enumerate() { - let idx = idx + 1; - match item { - Element::String(st) => write_str!(idx, st), - Element::RespCode(rc) => print_rcode(rc, Some(idx)), - Element::UnsignedInt(int) => write_int!(idx, int), - Element::Array(Array::Bin(brr)) => print_bin_array(brr), - Element::Array(Array::Str(srr)) => print_str_array(srr), - _ => eskysh!("Nested arrays cannot be printed just yet"), - } - } -} - -pub struct BinaryData(Vec); - -impl fmt::Display for BinaryData { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "b\"")?; - for b in self.0.iter() { - let b = *b; - // See this: https://doc.rust-lang.org/reference/tokens.html#byte-escapes - // this idea was borrowed from the Bytes crate - #[allow(clippy::manual_range_contains)] - if b == b'\n' { - write!(f, "\\n")?; - } else if b == b'\r' { - write!(f, "\\r")?; - } else if b == b'\t' { - write!(f, "\\t")?; - } else if b == b'\\' || b == b'"' { - write!(f, "\\{}", b as char)?; - } else if b == b'\0' { - write!(f, "\\0")?; - // ASCII printable - } else if b >= 0x20 && b < 0x7f { - write!(f, "{}", b as char)?; - } else { - write!(f, "\\x{:02x}", b)?; - } - } - write!(f, "\"")?; - Ok(()) - } -} diff --git a/cli/src/tests.rs b/cli/src/tests.rs deleted file mode 100644 index d527b1f8..00000000 --- a/cli/src/tests.rs +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Created on Sun Oct 10 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use crate::tokenizer::{get_query, TokenizerError}; - -fn query_from(input: &[u8]) -> Result, TokenizerError> { - get_query(input) -} - -#[test] -fn test_basic_tokenization() { - let input = "set x 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ); -} - -#[test] -fn test_single_quote_tokens() { - let input = "set 'x with a whitespace' 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "100".to_owned() - ] - ); -} - -#[test] -fn test_double_quote_tokens() { - let input = r#"set "x with a whitespace" 100"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "100".to_owned() - ] - ); -} - -#[test] -fn test_single_and_double_quote_tokens() { - let input = r#"set "x with a whitespace" 'y with a whitespace'"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_multiple_single_quote_tokens() { - let input = r#"'set' 'x with a whitespace' 'y with a whitespace'"#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_multiple_double_quote_tokens() { - let input = r#""set" "x with a whitespace" "y with a whitespace""#.as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec![ - "set".to_owned(), - "x with a whitespace".to_owned(), - "y with a whitespace".to_owned() - ] - ); -} - -#[test] -fn test_missing_single_quote() { - let input = r#"'get' 'x with a whitespace"#.as_bytes(); - let ret = format!("{}", query_from(input).unwrap_err()); - assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`"); -} - -#[test] -fn test_missing_double_quote() { - let input = r#"'get' "x with a whitespace"#.as_bytes(); - let ret = format!("{}", query_from(input).unwrap_err()); - assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`"); -} - -#[test] -fn test_extra_whitespace() { - let input = "set x '100'".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ); -} - -#[test] -fn test_singly_quoted() { - let input = "set tables' wth".as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("tables".to_owned())); -} - -#[test] -fn test_text_after_quote_nospace() { - let input = "get 'rust'ferris".as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust'".to_owned())); -} - -#[test] -fn test_text_after_double_quote_nospace() { - let input = r#"get "rust"ferris"#.as_bytes(); - let ret = query_from(input).unwrap_err(); - assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust\"".to_owned())); -} - -#[test] -fn test_inline_comment() { - let input = "set x 100 # sets x to 100".as_bytes(); - let ret = query_from(input).unwrap(); - assert_eq!( - ret, - vec!["set".to_owned(), "x".to_owned(), "100".to_owned()] - ) -} - -#[test] -fn test_full_comment() { - let input = "# what is going on?".as_bytes(); - let ret = query_from(input).unwrap(); - assert!(ret.is_empty()); -} - -#[test] -fn test_ignore_comment() { - let input = "set x \"# ooh la la\"".as_bytes(); - assert_eq!( - query_from(input).unwrap(), - vec!["set".to_owned(), "x".to_owned(), "# ooh la la".to_owned()] - ); - let input = "set x \"#\"".as_bytes(); - assert_eq!( - query_from(input).unwrap(), - vec!["set".to_owned(), "x".to_owned(), "#".to_owned()] - ); -} - -#[test] -fn test_blueql_query() { - let input = b"create model mymodel(string, binary)"; - assert_eq!( - query_from(input).unwrap(), - vec!["create model mymodel(string, binary)"] - ); -} diff --git a/cli/src/tokenizer.rs b/cli/src/tokenizer.rs deleted file mode 100644 index e6d6183f..00000000 --- a/cli/src/tokenizer.rs +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Created on Sat Oct 09 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! This module provides a simple way to avoid "the funk" with "funky input queries". It simply -//! tokenizes char-by-char analyzing quotes et al as required -//! - -use { - core::fmt, - skytable::{types::RawString, Query}, - std::collections::HashSet, -}; - -lazy_static::lazy_static! { - static ref BLUEQL_KW: HashSet<&'static [u8]> = { - let mut hs = HashSet::new(); - hs.insert("create".as_bytes()); - hs.insert(b"inspect"); - hs.insert(b"drop"); - hs.insert(b"use"); - hs - }; -} - -#[derive(Debug, PartialEq)] -pub enum TokenizerError { - QuoteMismatch(String), - BacktickMismatch(String), - ExpectedWhitespace(String), -} - -impl fmt::Display for TokenizerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::QuoteMismatch(expr) => write!(f, "mismatched quotes near end of: `{}`", expr), - Self::ExpectedWhitespace(expr) => { - write!(f, "expected whitespace near end of: `{}`", expr) - } - Self::BacktickMismatch(expr) => { - write!(f, "mismatched backticks near end of: `{}`", expr) - } - } - } -} - -pub trait SequentialQuery { - fn push(&mut self, input: &[u8]); - fn new() -> Self; - fn is_empty(&self) -> bool; -} - -// #[cfg(test)] -impl SequentialQuery for Vec { - fn push(&mut self, input: &[u8]) { - Vec::push(self, String::from_utf8_lossy(input).to_string()) - } - fn is_empty(&self) -> bool { - Vec::len(self) == 0 - } - fn new() -> Self { - Vec::new() - } -} - -impl SequentialQuery for Query { - fn push(&mut self, input: &[u8]) { - Query::push(self, RawString::from(input.to_owned())) - } - fn is_empty(&self) -> bool { - Query::len(self) == 0 - } - fn new() -> Self { - Query::new() - } -} - -// FIXME(@ohsayan): Fix this entire impl. At this point, it's almost like legacy code -pub fn get_query(inp: &[u8]) -> Result { - assert!(!inp.is_empty(), "Input is empty"); - let mut query = T::new(); - let mut it = inp.iter().peekable(); - macro_rules! pos { - () => { - inp.len() - it.len() - }; - } - macro_rules! expect_whitespace { - ($start:expr) => { - match it.peek() { - Some(b) => match **b { - b' ' => {} - _ => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(&inp[$start..pos!()]).to_string(), - )) - } - }, - None => {} - } - }; - } - // skip useless starting whitespace - while let Some(b' ') = it.next() {} - let end_of_first = match it.position(|x| *x == b' ') { - Some(e) => e + 1, - None if it.len() == 0 => inp.len(), - None => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(inp).to_string(), - )) - } - }; - if BLUEQL_KW.contains(inp[..end_of_first].to_ascii_lowercase().as_slice()) { - query.push(inp); - return Ok(query); - } else { - it = inp.iter().peekable(); - } - 'outer: while let Some(tok) = it.next() { - match tok { - b'\'' => { - // hmm, quotes; let's see where it ends - let pos = pos!(); - let qidx = it.position(|x| *x == b'\''); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::QuoteMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b'"' => { - // hmm, quotes; let's see where it ends - let pos = pos!(); - let qidx = it.position(|x| *x == b'"'); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::QuoteMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b'`' => { - // hmm, backtick? let's look for the end - let pos = pos!(); - let qidx = it.position(|x| *x == b'`'); - match qidx { - Some(idx) => query.push(&inp[pos..idx + pos]), - None => { - let end = pos!(); - return Err(TokenizerError::BacktickMismatch( - String::from_utf8_lossy(&inp[pos..end]).to_string(), - )); - } - } - expect_whitespace!(pos); - } - b' ' => { - // this just prevents control from being handed to the wildcard - continue; - } - b'#' => { - // so this is an inline comment; skip until newline - let _ = it.position(|x| *x == b'\n'); - } - _ => { - let start = pos!() - 1; - let mut end = start; - // alpha? cool, go on - 'inner: while let Some(tok) = it.peek() { - match **tok { - b' ' => { - it.next(); - break 'inner; - } - b'\'' | b'"' => { - return Err(TokenizerError::ExpectedWhitespace( - String::from_utf8_lossy(&inp[start..pos!()]).to_string(), - )) - } - b'#' => continue 'outer, - _ => { - end += 1; - it.next(); - continue 'inner; - } - } - } - end += 1; - query.push(&inp[start..end]); - } - } - } - Ok(query) -} diff --git a/examples/config-files/badcfg.toml b/examples/config-files/badcfg.toml deleted file mode 100644 index 4e7c7e47..00000000 --- a/examples/config-files/badcfg.toml +++ /dev/null @@ -1,3 +0,0 @@ -# This is a 'bad' configuration file since it contains an invalid port -[server] -port = 20033002 \ No newline at end of file diff --git a/examples/config-files/badcfg2.toml b/examples/config-files/badcfg2.toml deleted file mode 100644 index 066661b2..00000000 --- a/examples/config-files/badcfg2.toml +++ /dev/null @@ -1,8 +0,0 @@ -# This is a bad toml file since it contains an invalid value of `every` in BGSAVE -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -every = 0.5 # Not possible! -enabled = true \ No newline at end of file diff --git a/examples/config-files/bgsave-justenabled.toml b/examples/config-files/bgsave-justenabled.toml deleted file mode 100644 index b1dfbeb9..00000000 --- a/examples/config-files/bgsave-justenabled.toml +++ /dev/null @@ -1,6 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -enabled = true \ No newline at end of file diff --git a/examples/config-files/bgsave-justevery.toml b/examples/config-files/bgsave-justevery.toml deleted file mode 100644 index eb9e69a7..00000000 --- a/examples/config-files/bgsave-justevery.toml +++ /dev/null @@ -1,6 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -every = 600 \ No newline at end of file diff --git a/examples/config-files/docker.toml b/examples/config-files/docker.toml deleted file mode 100644 index 3b5058bd..00000000 --- a/examples/config-files/docker.toml +++ /dev/null @@ -1,5 +0,0 @@ -# this is the default configuration file for Docker images. Modify it as required -[server] -host = "0.0.0.0" -port = 2003 -noart = true diff --git a/examples/config-files/ipv6.toml b/examples/config-files/ipv6.toml deleted file mode 100644 index c98fe3bc..00000000 --- a/examples/config-files/ipv6.toml +++ /dev/null @@ -1,4 +0,0 @@ -# This makes use of an IPv6 address -[server] -host = "::1" -port = 2003 diff --git a/examples/config-files/secure-noart.toml b/examples/config-files/secure-noart.toml deleted file mode 100644 index 14adffca..00000000 --- a/examples/config-files/secure-noart.toml +++ /dev/null @@ -1,4 +0,0 @@ -[server] -host = '127.0.0.1' # i.e localhost -port = 2003 # The port to bind to -noart = true # No terminal artwork \ No newline at end of file diff --git a/examples/config-files/skyd.toml b/examples/config-files/skyd.toml deleted file mode 100644 index 7bf537d7..00000000 --- a/examples/config-files/skyd.toml +++ /dev/null @@ -1,5 +0,0 @@ -# This is a 'good' configuration file since it contains a valid port -# and appropriate keys -[server] -host = '127.0.0.1' -port = 2003 # Set the server's port to 2003 \ No newline at end of file diff --git a/examples/config-files/snapshot.toml b/examples/config-files/snapshot.toml deleted file mode 100644 index fd202c9f..00000000 --- a/examples/config-files/snapshot.toml +++ /dev/null @@ -1,17 +0,0 @@ -[server] -host = "127.0.0.1" # The IP address to which you want sdb to bind to -port = 2003 # The port to which you want sdb to bind to -# Set `noart` to true if you want to disable terminal artwork -noart = false - -[bgsave] -# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run -# after every 2 minutes -enabled = true -every = 120 - -[snapshot] -# Create a snapshot every hour (1 hour = 60 minutes = 60 * 60 seconds = 3600 seconds) -every = 3600 -# How many of the snapshots to keep -atmost = 4 # keep the four most recent snapshots diff --git a/examples/config-files/ssl.toml b/examples/config-files/ssl.toml deleted file mode 100644 index c4e951e3..00000000 --- a/examples/config-files/ssl.toml +++ /dev/null @@ -1,17 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 -noart = false - -[ssl] -key = "/path/to/keyfile.pem" -chain = "/path/to/chain.pem" -port = 2004 - -[bgsave] -enabled = true -every = 120 - -[snapshot] -every = 3600 -atmost = 4 \ No newline at end of file diff --git a/examples/config-files/template.toml b/examples/config-files/template.toml deleted file mode 100644 index bc21d894..00000000 --- a/examples/config-files/template.toml +++ /dev/null @@ -1,40 +0,0 @@ -# This is a complete sdb configuration template which is always kept updated -# to include all the configuration options. I encourage you to always use this -# when you use a configuration file -# Instead of deleting entire sections from this file, comment them out, so that you -# now what you've kept enabled and what you've kept disabled. This helps avoid -# configuration problems during production - -# This is a *REQUIRED* key -[server] -host = "127.0.0.1" # The IP address to which you want sdb to bind to -port = 2003 # The port to which you want sdb to bind to -noart = false # Set `noart` to true if you want to disable terminal artwork -maxcon = 50000 # set the maximum number of clients that the server can accept -mode = "dev" # Set this to `prod` when you're running in production and `dev` when in development - -# This is an optional key -[auth] -# the origin key to be used to claim the root account -origin_key = "4527387f92a381cbe804593f33991d327d456a97" - -# This key is *OPTIONAL* -[bgsave] -# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run -# after every 2 minutes -enabled = true -every = 120 - -# This key is *OPTIONAL* -[snapshot] -every = 3600 # Make a snapshot after every 1 hour (60min * 60sec= 3600secs) -atmost = 4 # Keep the 4 most recent snapshots -failsafe = true # stops accepting writes if snapshotting fails - -# This key is *OPTIONAL*, used for TLS/SSL config -[ssl] -key = "/path/to/keyfile.pem" -chain = "/path/to/chain.pem" -port = 2004 -only = true # optional to enable SSL-only requests -passin = "/path/to/cert/passphrase.txt" # optional to programmatically verify the TLS cert diff --git a/examples/config-files/template.yaml b/examples/config-files/template.yaml new file mode 100644 index 00000000..1e7f286c --- /dev/null +++ b/examples/config-files/template.yaml @@ -0,0 +1,22 @@ +system: + mode: prod + rs_window: 600 + +auth: + plugin: pwd + # replace with your root password of choice + root_pass: password + +endpoints: + secure: + host: 127.0.0.1 + port: 2004 + # replace `cert` with the path to your self-signed certificate + cert: cert.pem + # replace `private_key` with the path to your private key + private_key: private.key + # replace `passphrase.txt` with the path to your private key passphrase + pkey_passphrase: passphrase.txt + insecure: + host: 127.0.0.1 + port: 2003 diff --git a/examples/config-files/withcustombgsave.toml b/examples/config-files/withcustombgsave.toml deleted file mode 100644 index 34ee730b..00000000 --- a/examples/config-files/withcustombgsave.toml +++ /dev/null @@ -1,7 +0,0 @@ -[server] -host = "127.0.0.1" -port = 2003 - -[bgsave] -enabled = true -every = 600 # Every 10 minutes diff --git a/harness/Cargo.toml b/harness/Cargo.toml index 246f9243..605e03c9 100644 --- a/harness/Cargo.toml +++ b/harness/Cargo.toml @@ -7,9 +7,7 @@ edition = "2021" [dependencies] # internal deps -skytable = { git = "https://github.com/skytable/client-rust.git", features = [ - "sync", -], default-features = false } +skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" } libsky = { path = "../libsky" } # external deps env_logger = "0.10.0" diff --git a/harness/src/build.rs b/harness/src/build.rs index 2664e1ec..e897060a 100644 --- a/harness/src/build.rs +++ b/harness/src/build.rs @@ -34,7 +34,7 @@ use { }; /// The binaries that will be present in a bundle -pub const BINARIES: [&str; 4] = ["skyd", "sky-bench", "skysh", "sky-migrate"]; +pub const BINARIES: [&str; 3] = ["skyd", "sky-bench", "skysh"]; /// The build mode #[derive(Copy, Clone, PartialEq)] diff --git a/harness/src/test/mod.rs b/harness/src/test/mod.rs index ae2afda1..2695e109 100644 --- a/harness/src/test/mod.rs +++ b/harness/src/test/mod.rs @@ -35,8 +35,9 @@ use { bn::{BigNum, MsbOption}, error::ErrorStack, hash::MessageDigest, - pkey::{PKey, Private}, + pkey::PKey, rsa::Rsa, + symm::Cipher, x509::{ extension::{BasicConstraints, KeyUsage, SubjectKeyIdentifier}, X509NameBuilder, X509, @@ -89,16 +90,16 @@ fn append_target(args: &mut Vec) { /// - The standard test suite /// - The persistence test suite fn run_test_inner() -> HarnessResult<()> { + const TEST_PASSWORD: &str = "xCqe4yuVM7l2MnHZOFZDDieqjqmmL3qvO5LOEOhpXPE="; // first create the TLS keys info!("Creating TLS key+cert"); - let (cert, pkey) = mk_ca_cert().expect("Failed to create cert"); + let (cert, pkey) = mk_ca_cert(TEST_PASSWORD.as_bytes()).expect("Failed to create cert"); + let mut passfile = fs::File::create("passphrase.txt").unwrap(); + passfile.write_all(TEST_PASSWORD.as_bytes()).unwrap(); let mut certfile = fs::File::create("cert.pem").expect("failed to create cert.pem"); - certfile.write_all(&cert.to_pem().unwrap()).unwrap(); + certfile.write_all(&cert).unwrap(); let mut pkeyfile = fs::File::create("key.pem").expect("failed to create key.pem"); - pkeyfile - .write_all(&pkey.private_key_to_pem_pkcs8().unwrap()) - .unwrap(); - + pkeyfile.write_all(&pkey).unwrap(); // assemble commands let target_folder = util::get_target_folder(BuildMode::Debug); let mut standard_test_suite_args = vec!["cargo".to_owned(), "test".into()]; @@ -110,15 +111,9 @@ fn run_test_inner() -> HarnessResult<()> { ]; append_target(&mut build_cmd_args); append_target(&mut standard_test_suite_args); - let persist_test_suite_args = [ - standard_test_suite_args.as_slice(), - &["--features".to_owned(), "persist-suite".into()], - ] - .concat(); // get cmd let build_cmd = util::assemble_command_from_slice(build_cmd_args); let standard_test_suite = util::assemble_command_from_slice(standard_test_suite_args); - let persist_test_suite = util::assemble_command_from_slice(persist_test_suite_args); // build skyd info!("Building server binary ..."); @@ -130,20 +125,11 @@ fn run_test_inner() -> HarnessResult<()> { util::handle_child("standard test suite", standard_test_suite)?; Ok(()) })?; - - // run persistence tests; don't kill the servers because if either of the tests fail - // then we can ensure that they will be killed by `run_test` - svc::run_with_servers(&target_folder, false, move || { - info!("Running persistence test suite ..."); - util::handle_child("standard test suite", persist_test_suite)?; - Ok(()) - })?; - Ok(()) } /// Generate certificates -fn mk_ca_cert() -> Result<(X509, PKey), ErrorStack> { +fn mk_ca_cert(password: &[u8]) -> Result<(Vec, Vec), ErrorStack> { let rsa = Rsa::generate(2048)?; let key_pair = PKey::from_rsa(rsa)?; @@ -182,9 +168,8 @@ fn mk_ca_cert() -> Result<(X509, PKey), ErrorStack> { let subject_key_identifier = SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?; cert_builder.append_extension(subject_key_identifier)?; - cert_builder.sign(&key_pair, MessageDigest::sha256())?; - let cert = cert_builder.build(); - + let cert = cert_builder.build().to_pem().unwrap(); + let key_pair = key_pair.private_key_to_pem_pkcs8_passphrase(Cipher::des_cbc(), password)?; Ok((cert, key_pair)) } diff --git a/harness/src/test/svc.rs b/harness/src/test/svc.rs index 0c583969..433e0fcd 100644 --- a/harness/src/test/svc.rs +++ b/harness/src/test/svc.rs @@ -26,12 +26,16 @@ #[cfg(windows)] use std::os::windows::process::CommandExt; + use { crate::{ util::{self}, HarnessError, HarnessResult, ROOT_DIR, }, - skytable::{error::Error, Connection, SkyResult}, + skytable::{ + error::{ClientResult, Error}, + Config, Connection, + }, std::{ io::ErrorKind, path::Path, @@ -45,16 +49,17 @@ const POWERSHELL_SCRIPT: &str = include_str!("../../../ci/windows/stop.ps1"); #[cfg(windows)] /// Flag for new console Window const CREATE_NEW_CONSOLE: u32 = 0x00000010; -pub(super) const SERVERS: [(&str, [u16; 2]); 3] = [ - ("server1", [2003, 2004]), - ("server2", [2005, 2006]), - ("server3", [2007, 2008]), -]; +pub(super) const SERVERS: [(&str, [u16; 2]); 1] = [("server1", [2003, 2004])]; /// The test suite server host const TESTSUITE_SERVER_HOST: &str = "127.0.0.1"; /// The workspace root const WORKSPACE_ROOT: &str = env!("ROOT_DIR"); +fn connect_db(host: &str, port: u16) -> ClientResult { + let cfg = Config::new(host, port, "root", "password12345678"); + cfg.connect() +} + /// Get the command to start the provided server1 pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef) -> Command { let args = vec![ @@ -63,8 +68,8 @@ pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef HarnessResult<()> { Ok(()) } -fn connection_refused(input: SkyResult) -> HarnessResult { +fn connection_refused(input: ClientResult) -> HarnessResult { match input { Ok(_) => Ok(false), Err(Error::IoError(e)) @@ -118,7 +123,7 @@ fn wait_for_startup() -> HarnessResult<()> { for port in ports { let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}"); let mut backoff = 1; - let mut con = Connection::new(TESTSUITE_SERVER_HOST, port); + let mut con = connect_db(TESTSUITE_SERVER_HOST, port); while connection_refused(con)? { if backoff > 64 { // enough sleeping, return an error @@ -131,7 +136,7 @@ fn wait_for_startup() -> HarnessResult<()> { "Server at {connection_string} not started. Sleeping for {backoff} second(s) ..." ); util::sleep_sec(backoff); - con = Connection::new(TESTSUITE_SERVER_HOST, port); + con = connect_db(TESTSUITE_SERVER_HOST, port); backoff *= 2; } info!("Server at {connection_string} has started"); @@ -148,7 +153,7 @@ fn wait_for_shutdown() -> HarnessResult<()> { for port in ports { let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}"); let mut backoff = 1; - let mut con = Connection::new(TESTSUITE_SERVER_HOST, port); + let mut con = connect_db(TESTSUITE_SERVER_HOST, port); while !connection_refused(con)? { if backoff > 64 { // enough sleeping, return an error @@ -161,7 +166,7 @@ fn wait_for_shutdown() -> HarnessResult<()> { "Server at {connection_string} still active. Sleeping for {backoff} second(s) ..." ); util::sleep_sec(backoff); - con = Connection::new(TESTSUITE_SERVER_HOST, port); + con = connect_db(TESTSUITE_SERVER_HOST, port); backoff *= 2; } info!("Server at {connection_string} has stopped accepting connections"); diff --git a/pkg/debian/description.txt b/pkg/debian/description.txt index 15a8dca7..ac8d681f 100644 --- a/pkg/debian/description.txt +++ b/pkg/debian/description.txt @@ -1,5 +1,5 @@ Skytable is a free and open-source NoSQL database that aims to provide flexibility in data modeling at scale. The `skytable` package contains the database server (`skyd`), -an interactive command-line client (`skysh`), a benchmarking -tool (`sky-bench`) and a migration tool (`sky-migrate`). \ No newline at end of file +an interactive command-line client (`skysh`) and a benchmarking +tool (`sky-bench`). \ No newline at end of file diff --git a/server/Cargo.toml b/server/Cargo.toml index e218fcbc..d99e8c5a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,7 +18,7 @@ openssl = { version = "0.10.56", features = ["vendored"] } crossbeam-epoch = { version = "0.9.15" } parking_lot = "0.12.1" serde = { version = "1.0.183", features = ["derive"] } -tokio = { version = "1.30.0", features = ["full"] } +tokio = { version = "1.34.0", features = ["full"] } tokio-openssl = "0.6.3" uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] } crc = "3.0.1" @@ -42,7 +42,7 @@ cc = "1.0.82" [dev-dependencies] # external deps rand = "0.8.5" -tokio = { version = "1.30.0", features = ["test-util"] } +tokio = { version = "1.34.0", features = ["test-util"] } [features] nightly = [] diff --git a/server/src/cli.yml b/server/src/cli.yml deleted file mode 100644 index c8a3024c..00000000 --- a/server/src/cli.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Skytable Server -version: 0.8.0 -author: Sayan N. -about: The Skytable Database server -args: - - config: - short: c - required: false - long: withconfig - value_name: cfgfile - help: Sets a configuration file to start skyd - takes_value: true - - restore: - short: r - required: false - long: restore - value_name: backupdir - help: Restores data from a previous snapshot made in the provided directory - takes_value: true - - host: - short: h - required: false - long: host - value_name: host - help: Sets the host to which the server will bind - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the port to which the server will bind - takes_value: true - - noart: - required: false - long: noart - help: Disables terminal artwork - takes_value: false - - nosave: - required: false - long: nosave - help: Disables automated background saving - takes_value: false - - saveduration: - required: false - long: saveduration - value_name: duration - short: S - takes_value: true - help: Set the BGSAVE duration - - snapevery: - required: false - long: snapevery - value_name: duration - help: Set the periodic snapshot duration - takes_value: true - - snapkeep: - required: false - long: snapkeep - value_name: count - help: Sets the number of most recent snapshots to keep - takes_value: true - - sslkey: - required: false - long: sslkey - short: k - value_name: key - help: Sets the PEM key file to use for SSL/TLS - takes_value: true - - sslchain: - required: false - long: sslchain - short: z - value_name: chain - help: Sets the PEM chain file to use for SSL/TLS - takes_value: true - - sslonly: - required: false - long: sslonly - takes_value: false - help: Tells the server to only accept SSL connections and disables the non-SSL port - - sslport: - required: false - long: sslport - takes_value: true - value_name: sslport - help: Set a custom SSL port to bind to - - tlspassin: - required: false - long: tlspassin - takes_value: true - value_name: tlspassin - help: Path to the file containing the passphrase for the TLS certificate - - stopwriteonfail: - required: false - long: stop-write-on-fail - takes_value: true - help: Stop accepting writes if any persistence method except BGSAVE fails (defaults to true) - - maxcon: - required: false - long: maxcon - takes_value: true - help: Set the maximum number of connections - value_name: maxcon - - mode: - required: false - long: mode - takes_value: true - short: m - help: Sets the deployment type - value_name: mode - - authkey: - required: false - long: auth-origin-key - takes_value: true - help: Set the authentication origin key - value_name: origin_key - - protover: - required: false - long: protover - takes_value: true - help: Set the protocol version - value_name: protover diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs index 93c44647..330d0639 100644 --- a/server/src/engine/config.rs +++ b/server/src/engine/config.rs @@ -25,7 +25,7 @@ */ use { - crate::engine::error::RuntimeResult, + crate::engine::{error::RuntimeResult, fractal}, core::fmt, serde::Deserialize, std::{collections::HashMap, fs}, @@ -97,7 +97,6 @@ impl Configuration { } const DEFAULT_HOST: &'static str = "127.0.0.1"; const DEFAULT_PORT_TCP: u16 = 2003; - const DEFAULT_RELIABILITY_SVC_PING: u64 = 5 * 60; pub fn default_dev_mode(auth: DecodedAuth) -> Self { Self { endpoints: ConfigEndpoint::Insecure(ConfigEndpointTcp { @@ -105,9 +104,7 @@ impl Configuration { port: Self::DEFAULT_PORT_TCP, }), mode: ConfigMode::Dev, - system: ConfigSystem { - reliability_system_window: Self::DEFAULT_RELIABILITY_SVC_PING, - }, + system: ConfigSystem::new(fractal::GENERAL_EXECUTOR_WINDOW), auth: ConfigAuth::new(auth.plugin, auth.root_pass), } } @@ -204,11 +201,10 @@ pub enum ConfigMode { /// System configuration settings pub struct ConfigSystem { /// time window in seconds for the reliability system to kick-in automatically - reliability_system_window: u64, + pub reliability_system_window: u64, } impl ConfigSystem { - #[cfg(test)] pub fn new(reliability_system_window: u64) -> Self { Self { reliability_system_window, @@ -472,6 +468,7 @@ fn decode_tls_ep( host: &str, port: u16, ) -> RuntimeResult { + super::fractal::context::set_dmsg("loading TLS configuration from disk"); let tls_key = fs::read_to_string(key_path)?; let tls_cert = fs::read_to_string(cert_path)?; let tls_priv_key_passphrase = fs::read_to_string(pkey_pass)?; @@ -653,6 +650,7 @@ Flags: -v, --version Display the version number and exit. Options: + --config Set configuration options using the config file --tlscert Specify the path to the TLS certificate. --tlskey Specify the path to the TLS private key. --endpoint Designate an endpoint. Format: protocol@host:port. @@ -1075,6 +1073,7 @@ fn get_file_from_store(filename: &str) -> RuntimeResult { } #[cfg(not(test))] { + super::fractal::context::set_dmsg("loading configuration file from disk"); f = Ok(fs::read_to_string(filename)?); } f @@ -1202,6 +1201,12 @@ fn check_config_file( match config_from_file.endpoints.as_mut() { Some(ep) => match ep.secure.as_mut() { Some(secure_ep) => { + super::fractal::context::set_dmsg("loading TLS configuration from disk"); + dbg!( + &secure_ep.cert, + &secure_ep.private_key, + &secure_ep.pkey_passphrase + ); let cert = fs::read_to_string(&secure_ep.cert)?; let private_key = fs::read_to_string(&secure_ep.private_key)?; let private_key_passphrase = fs::read_to_string(&secure_ep.pkey_passphrase)?; diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs index c5c1104b..4446a9e5 100644 --- a/server/src/engine/core/exec.rs +++ b/server/src/engine/core/exec.rs @@ -187,6 +187,12 @@ fn blocking_exec_sysctl( let userdel = ASTNode::from_state(&mut state)?; super::dcl::drop_user(&g, cstate, userdel) } + (Token::Ident(k1), Token::Ident(k2)) + if k1.eq_ignore_ascii_case("report") && k2.eq_ignore_ascii_case("status") => + { + // TODO(@ohsayan): replace dummy endpoint with actual `system report status` responses + Ok(()) + } _ => Err(QueryError::QLUnknownStatement), } } diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs index 1cabe3eb..63906601 100644 --- a/server/src/engine/fractal/mgr.rs +++ b/server/src/engine/fractal/mgr.rs @@ -46,6 +46,8 @@ use { }, }; +pub const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60; + /// A task for the [`FractalMgr`] to perform pub struct Task { threshold: usize, @@ -190,13 +192,13 @@ impl FractalBoot { hp_recv, } } - pub fn boot(self, sigterm: &broadcast::Sender<()>) -> FractalHandle { + pub fn boot(self, sigterm: &broadcast::Sender<()>, rs_window: u64) -> FractalHandle { let Self { global, lp_recv: lp_receiver, hp_recv: hp_receiver, } = self; - FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver) + FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver, rs_window) } } @@ -207,6 +209,7 @@ impl FractalMgr { sigterm: &broadcast::Sender<()>, lp_receiver: UnboundedReceiver>, hp_receiver: UnboundedReceiver>, + rs_window: u64, ) -> FractalHandle { let fractal_mgr = global.get_state().fractal_mgr(); let global_1 = global.clone(); @@ -217,7 +220,14 @@ impl FractalMgr { }); let sigterm_rx = sigterm.subscribe(); let lp_handle = tokio::spawn(async move { - FractalMgr::general_executor_svc(fractal_mgr, global_2, lp_receiver, sigterm_rx).await + FractalMgr::general_executor_svc( + fractal_mgr, + global_2, + lp_receiver, + sigterm_rx, + rs_window, + ) + .await }); FractalHandle { hp_handle, @@ -228,7 +238,6 @@ impl FractalMgr { // services impl FractalMgr { - const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60; /// The high priority executor service runs in the background to take care of high priority tasks and take any /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is /// allowed to perform HP tasks @@ -323,7 +332,9 @@ impl FractalMgr { global: super::Global, mut lpq: UnboundedReceiver>, mut sigterm: broadcast::Receiver<()>, + rs_window: u64, ) { + let dur = std::time::Duration::from_secs(rs_window); loop { tokio::select! { _ = sigterm.recv() => { @@ -333,7 +344,7 @@ impl FractalMgr { info!("flp: exited executor service"); break; }, - _ = tokio::time::sleep(std::time::Duration::from_secs(Self::GENERAL_EXECUTOR_WINDOW)) => { + _ = tokio::time::sleep(dur) => { let global = global.clone(); tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap() } diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs index 74975e37..1645f2e6 100644 --- a/server/src/engine/fractal/mod.rs +++ b/server/src/engine/fractal/mod.rs @@ -51,7 +51,7 @@ pub mod test_utils; mod util; pub use { drivers::FractalModelDriver, - mgr::{CriticalTask, GenericTask, Task}, + mgr::{CriticalTask, GenericTask, Task, GENERAL_EXECUTOR_WINDOW}, util::FractalToken, }; diff --git a/server/src/engine/fractal/sys_store.rs b/server/src/engine/fractal/sys_store.rs index 625285d0..7ec79546 100644 --- a/server/src/engine/fractal/sys_store.rs +++ b/server/src/engine/fractal/sys_store.rs @@ -207,7 +207,7 @@ impl SysAuth { pub fn new(users: HashMap, SysAuthUser>) -> Self { Self { users } } - pub fn verify_user_is_root + ?Sized>( + pub fn verify_user_check_root + ?Sized>( &self, username: &str, password: &T, @@ -225,7 +225,7 @@ impl SysAuth { username: &str, password: &T, ) -> QueryResult<()> { - self.verify_user_is_root(username, password).map(|_| ()) + self.verify_user_check_root(username, password).map(|_| ()) } pub fn users(&self) -> &HashMap, SysAuthUser> { &self.users diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs index bb8f7788..73f75835 100644 --- a/server/src/engine/mod.rs +++ b/server/src/engine/mod.rs @@ -144,14 +144,16 @@ impl EndpointListeners { pub async fn start( termsig: TerminationSignal, - Configuration { endpoints, .. }: Configuration, + Configuration { + endpoints, system, .. + }: Configuration, fractal::GlobalStateStart { global, boot }: fractal::GlobalStateStart, ) -> RuntimeResult<()> { // create our system-wide channel let (signal, _) = broadcast::channel::<()>(1); // start our services context::set_dmsg("starting fractal engine"); - let fractal_handle = boot.boot(&signal); + let fractal_handle = boot.boot(&signal, system.reliability_system_window); // create our server context::set(Subsystem::Network, "initializing endpoints"); let str; diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs index a553d5db..c539309c 100644 --- a/server/src/engine/net/protocol/mod.rs +++ b/server/src/engine/net/protocol/mod.rs @@ -217,7 +217,7 @@ async fn do_handshake( match core::str::from_utf8(handshake.hs_auth().username()) { Ok(uname) => { let auth = global.sys_store().system_store().auth_data().read(); - let r = auth.verify_user_is_root(uname, handshake.hs_auth().password()); + let r = auth.verify_user_check_root(uname, handshake.hs_auth().password()); match r { Ok(is_root) => { let hs = handshake.hs_static(); diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs index b7f13946..c32ca03e 100644 --- a/server/src/engine/net/protocol/tests.rs +++ b/server/src/engine/net/protocol/tests.rs @@ -39,7 +39,7 @@ use crate::engine::{ client handshake */ -const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x015\n8\nsayanpass1234"; +const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x005\n8\nsayanpass1234"; const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new( HandshakeVersion::Original, diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs index 75559a7f..7ec18033 100644 --- a/server/src/engine/txn/gns/tests/full_chain.rs +++ b/server/src/engine/txn/gns/tests/full_chain.rs @@ -83,7 +83,9 @@ fn create_space() { &*space, &Space::new_restore_empty( uuid, - into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536))) + into_dict!("env" => DictEntryGeneric::Map( + into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536))) + )) ) ); }) @@ -111,7 +113,9 @@ fn alter_space() { &*space, &Space::new_restore_empty( uuid, - into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536))) + into_dict!("env" => DictEntryGeneric::Map( + into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)) + ))) ) ); }) diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml index 8726792b..19595da4 100644 --- a/sky-bench/Cargo.toml +++ b/sky-bench/Cargo.toml @@ -8,16 +8,5 @@ version = "0.8.0" [dependencies] # internal deps -skytable = { git = "https://github.com/skytable/client-rust.git", features = [ - "sync", - "dbg", -] } +skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" } libstress = { path = "../libstress" } -# external deps -clap = { version = "2", features = ["yaml"] } -log = "0.4.19" -env_logger = "0.10.0" -devtimer = "4.0.1" -serde = { version = "1.0.183", features = ["derive"] } -serde_json = "1.0.104" -rand = "0.8.5" diff --git a/sky-bench/src/bench/benches.rs b/sky-bench/src/bench/benches.rs deleted file mode 100644 index 87dc93d4..00000000 --- a/sky-bench/src/bench/benches.rs +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Created on Sat Aug 13 2022 - * - * This file is a part of S{ - let ref this = loopmon; - this.current -}le (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::{ - report::{AggregateReport, SingleReport}, - validation, vec_with_cap, BenchmarkConfig, LoopMonitor, - }, - crate::error::BResult, - devtimer::SimpleTimer, - libstress::Workpool, - skytable::{types::RawString, Connection, Element, Query, RespCode}, - std::{ - io::{Read, Write}, - net::{Shutdown, TcpStream}, - }, -}; - -/// Run a benchmark using the given pre-loop, in-loop and post-loop closures -fn run_bench_custom( - bench_config: BenchmarkConfig, - packets: Vec>, - on_init: Lv, - on_loop: Lp, - on_loop_exit: Ex, - loopmon: LoopMonitor, - reports: &mut AggregateReport, -) -> BResult<()> -where - Ex: Clone + Fn(&mut Inp) + Send + Sync + 'static, - Inp: Sync + 'static, - Lp: Clone + Fn(&mut Inp, Box<[u8]>) + Send + Sync + 'static, - Lv: Clone + Fn() -> Inp + Send + 'static + Sync, -{ - // now do our runs - let mut loopmon = loopmon; - - while loopmon.should_continue() { - // now create our connection pool - let pool = Workpool::new( - bench_config.server.connections(), - on_init.clone(), - on_loop.clone(), - on_loop_exit.clone(), - true, - Some(bench_config.query_count()), - )?; - - // get our local copy - let this_packets = packets.clone(); - - // run and time our operations - let mut dt = SimpleTimer::new(); - dt.start(); - pool.execute_and_finish_iter(this_packets); - dt.stop(); - loopmon.incr_time(&dt); - - // cleanup - loopmon.cleanup()?; - loopmon.step(); - } - - // save time - reports.push(SingleReport::new( - loopmon.name(), - loopmon.sum() as f64 / bench_config.runs() as f64, - )); - Ok(()) -} - -#[inline(always)] -/// Init connection and buffer -fn init_connection_and_buf( - host: &str, - port: u16, - start_command: Vec, - bufsize: usize, -) -> (TcpStream, Vec) { - let mut con = TcpStream::connect((host, port)).unwrap(); - con.write_all(&start_command).unwrap(); - let mut ret = [0u8; validation::RESPCODE_OKAY.len()]; - con.read_exact(&mut ret).unwrap(); - let readbuf = vec![0; bufsize]; - (con, readbuf) -} - -/// Benchmark SET -pub fn bench_set( - keys: &[Vec], - values: &[Vec], - connection: &mut Connection, - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new_cleanup( - bench_config.runs(), - "set", - connection, - Query::from("FLUSHDB").arg("default.tmpbench"), - Element::RespCode(RespCode::Okay), - true, - ); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("SET") - .arg(RawString::from(keys[i].to_owned())) - .arg(RawString::from(values[i].to_owned())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::RESPCODE_OKAY.len(), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - assert_eq!(buf, validation::RESPCODE_OKAY); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} - -/// Benchmark UPDATE -pub fn bench_update( - keys: &[Vec], - new_value: &[u8], - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new(bench_config.runs(), "update"); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("update") - .arg(RawString::from(keys[i].clone())) - .arg(RawString::from(new_value.to_owned())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::RESPCODE_OKAY.len(), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - assert_eq!(buf, validation::RESPCODE_OKAY); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} - -/// Benchmark GET -pub fn bench_get( - keys: &[Vec], - bench_config: &BenchmarkConfig, - create_table: &[u8], - reports: &mut AggregateReport, -) -> BResult<()> { - let bench_config = bench_config.clone(); - let create_table = create_table.to_owned(); - let loopmon = LoopMonitor::new(bench_config.runs(), "get"); - let mut packets = vec_with_cap(bench_config.query_count())?; - (0..bench_config.query_count()).for_each(|i| { - packets.push( - Query::from("get") - .arg(RawString::from(keys[i].clone())) - .into_raw_query() - .into_boxed_slice(), - ) - }); - run_bench_custom( - bench_config.clone(), - packets, - move || { - init_connection_and_buf( - bench_config.server.host(), - bench_config.server.port(), - create_table.to_owned(), - validation::calculate_response_size(bench_config.kvsize()), - ) - }, - |(con, buf), packet| { - con.write_all(&packet).unwrap(); - con.read_exact(buf).unwrap(); - }, - |(con, _)| con.shutdown(Shutdown::Both).unwrap(), - loopmon, - reports, - ) -} diff --git a/sky-bench/src/bench/mod.rs b/sky-bench/src/bench/mod.rs deleted file mode 100644 index 77cb2789..00000000 --- a/sky-bench/src/bench/mod.rs +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Created on Tue Aug 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - self::report::AggregateReport, - crate::{ - config, - config::{BenchmarkConfig, ServerConfig}, - error::{BResult, Error}, - util, - }, - clap::ArgMatches, - devtimer::SimpleTimer, - libstress::utils::{generate_random_byte_vector, ran_bytes}, - skytable::{Connection, Element, Query, RespCode}, -}; - -mod benches; -mod report; -mod validation; - -macro_rules! binfo { - ($($arg:tt)+) => { - if $crate::config::should_output_messages() { - ::log::info!($($arg)+) - } - }; -} - -/// The loop monitor can be used for maintaining a loop for a given benchmark -struct LoopMonitor<'a> { - /// cleanup instructions - inner: Option>, - /// maximum iterations - max: usize, - /// current iteration - current: usize, - /// total time - time: u128, - /// name of test - name: &'static str, -} - -impl<'a> LoopMonitor<'a> { - /// Create a benchmark loop monitor that doesn't need any cleanup - pub fn new(max: usize, name: &'static str) -> Self { - Self { - inner: None, - max, - current: 0, - time: 0, - name, - } - } - /// Create a new benchmark loop monitor that uses the given cleanup instructions: - /// - `max`: Total iterations - /// - `name`: Name of benchmark - /// - `connection`: A connection to use for cleanup instructions - /// - `query`: Query to run for cleanup - /// - `response`: Response expected when cleaned up - /// - `skip_on_last`: Skip running the cleanup instructions on the last loop - pub fn new_cleanup( - max: usize, - name: &'static str, - connection: &'a mut Connection, - query: Query, - response: Element, - skip_on_last: bool, - ) -> Self { - Self { - inner: Some(CleanupInner::new(query, response, connection, skip_on_last)), - max, - current: 0, - time: 0, - name, - } - } - /// Run cleanup - fn cleanup(&mut self) -> BResult<()> { - let last_iter = self.is_last_iter(); - if let Some(ref mut cleanup) = self.inner { - let should_run_cleanup = !(last_iter && cleanup.skip_on_last); - if should_run_cleanup { - return cleanup.cleanup(self.name); - } - } - Ok(()) - } - /// Check if this is the last iteration - fn is_last_iter(&self) -> bool { - (self.max - 1) == self.current - } - /// Step the counter ahead - fn step(&mut self) { - self.current += 1; - } - /// Determine if we should continue executing - fn should_continue(&self) -> bool { - self.current < self.max - } - /// Append a new time to the sum - fn incr_time(&mut self, dt: &SimpleTimer) { - self.time += dt.time_in_nanos().unwrap(); - } - /// Return the sum - fn sum(&self) -> u128 { - self.time - } - /// Return the name of the benchmark - fn name(&self) -> &'static str { - self.name - } -} - -/// Cleanup instructions -struct CleanupInner<'a> { - /// the connection to use for cleanup processes - connection: &'a mut Connection, - /// the query to be run - query: Query, - /// the response to expect - response: Element, - /// whether we should skip on the last loop - skip_on_last: bool, -} - -impl<'a> CleanupInner<'a> { - /// Init cleanup instructions - fn new(q: Query, r: Element, connection: &'a mut Connection, skip_on_last: bool) -> Self { - Self { - query: q, - response: r, - connection, - skip_on_last, - } - } - /// Run cleanup - fn cleanup(&mut self, name: &'static str) -> BResult<()> { - let r: Element = self.connection.run_query(&self.query)?; - if r.ne(&self.response) { - Err(Error::Runtime(format!( - "Failed to run cleanup for benchmark `{}`", - name - ))) - } else { - Ok(()) - } - } -} - -#[inline(always)] -/// Returns a vec with the given cap, ensuring that we don't overflow memory -fn vec_with_cap(cap: usize) -> BResult> { - let mut v = Vec::new(); - v.try_reserve_exact(cap)?; - Ok(v) -} - -/// Run the actual benchmarks -pub fn run_bench(servercfg: &ServerConfig, matches: ArgMatches) -> BResult<()> { - // init bench config - let bench_config = BenchmarkConfig::new(servercfg, matches)?; - // check if we have enough combinations for the given query count and key size - if !util::has_enough_ncr(bench_config.kvsize(), bench_config.query_count()) { - return Err(Error::Runtime( - "too low sample space for given query count. use larger kvsize".into(), - )); - } - // run sanity test; this will also set up the temporary table for benchmarking - binfo!("Running sanity test ..."); - util::run_sanity_test(&bench_config.server)?; - - // pool pre-exec setup - let servercfg = servercfg.clone(); - let switch_table = Query::from("use default.tmpbench").into_raw_query(); - - // init pool config; side_connection is for cleanups - let mut misc_connection = Connection::new(servercfg.host(), servercfg.port())?; - - // init timer and reports - let mut reports = AggregateReport::new(bench_config.query_count()); - - // init test data - binfo!("Initializing test data ..."); - let mut rng = rand::thread_rng(); - let keys = generate_random_byte_vector( - bench_config.query_count(), - bench_config.kvsize(), - &mut rng, - true, - )?; - let values = generate_random_byte_vector( - bench_config.query_count(), - bench_config.kvsize(), - &mut rng, - false, - )?; - let new_updated_key = ran_bytes(bench_config.kvsize(), &mut rng); - - // run tests; the idea here is to run all tests one-by-one instead of generating all packets at once - // such an approach helps us keep memory usage low - // bench set - binfo!("Benchmarking SET ..."); - benches::bench_set( - &keys, - &values, - &mut misc_connection, - &bench_config, - &switch_table, - &mut reports, - )?; - - // bench update - binfo!("Benchmarking UPDATE ..."); - benches::bench_update( - &keys, - &new_updated_key, - &bench_config, - &switch_table, - &mut reports, - )?; - - // bench get - binfo!("Benchmarking GET ..."); - benches::bench_get(&keys, &bench_config, &switch_table, &mut reports)?; - - // remove all test data - binfo!("Finished benchmarks. Cleaning up ..."); - let r: Element = misc_connection.run_query(Query::from("drop model default.tmpbench force"))?; - if r != Element::RespCode(RespCode::Okay) { - return Err(Error::Runtime("failed to clean up after benchmarks".into())); - } - - if config::should_output_messages() { - // normal output - println!("===========RESULTS==========="); - let (maxpad, reports) = reports.finish(); - for report in reports { - let padding = " ".repeat(maxpad - report.name().len()); - println!( - "{}{} {:.6}/sec", - report.name().to_uppercase(), - padding, - report.stat(), - ); - } - println!("============================="); - } else { - // JSON - println!("{}", reports.into_json()) - } - Ok(()) -} diff --git a/sky-bench/src/bench/report.rs b/sky-bench/src/bench/report.rs deleted file mode 100644 index 4d0edbab..00000000 --- a/sky-bench/src/bench/report.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Created on Wed Aug 10 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use serde::Serialize; - -#[derive(Serialize)] -pub struct SingleReport { - name: &'static str, - stat: f64, -} - -impl SingleReport { - pub fn new(name: &'static str, stat: f64) -> Self { - Self { name, stat } - } - - pub fn stat(&self) -> f64 { - self.stat - } - - pub fn name(&self) -> &str { - self.name - } -} - -pub struct AggregateReport { - names: Vec, - query_count: usize, -} - -impl AggregateReport { - pub fn new(query_count: usize) -> Self { - Self { - names: Vec::new(), - query_count, - } - } - pub fn push(&mut self, report: SingleReport) { - self.names.push(report) - } - pub(crate) fn into_json(self) -> String { - let (_, report) = self.finish(); - serde_json::to_string(&report).unwrap() - } - - pub(crate) fn finish(self) -> (usize, Vec) { - let mut maxpad = self.names[0].name.len(); - let mut reps = self.names; - reps.iter_mut().for_each(|rep| { - let total_time = rep.stat; - let qps = (self.query_count as f64 / total_time) * 1_000_000_000_f64; - rep.stat = qps; - if rep.name.len() > maxpad { - maxpad = rep.name.len(); - } - }); - (maxpad, reps) - } -} diff --git a/sky-bench/src/bench/validation.rs b/sky-bench/src/bench/validation.rs deleted file mode 100644 index c93b35f0..00000000 --- a/sky-bench/src/bench/validation.rs +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Created on Tue Aug 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -pub const RESPCODE_OKAY: &[u8] = b"*!0\n"; - -pub fn calculate_response_size(keylen: usize) -> usize { - /* - *+5\n - hello - */ - let mut size = 2; // simple query byte + tsymbol - size += keylen.to_string().len(); // bytes in length - size += 1; // LF - size += keylen; // payload - size -} diff --git a/sky-bench/src/cli.yml b/sky-bench/src/cli.yml deleted file mode 100644 index 0afc683c..00000000 --- a/sky-bench/src/cli.yml +++ /dev/null @@ -1,72 +0,0 @@ -# -# Created on Tue Nov 03 2020 -# -# This file is a part of Skytable -# Copyright (c) 2020, Sayan Nandan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# - -name: Skytable Benchmark Tool -version: 0.8.0 -author: Sayan N. -about: | - The Skytable benchmark tool can be used to benchmark Skytable installations. - If you find any issues, then report one here: https://github.com/skytable/skytable -args: - - connections: - short: c - long: connections - value_name: count - help: Sets the number of simultaneous clients - takes_value: true - - queries: - short: q - long: queries - value_name: number - help: Sets the number of queries to run - takes_value: true - - size: - short: s - long: kvsize - value_name: bytes - help: Sets the size of the key/value pairs - takes_value: true - - json: - required: false - long: json - help: Sets output type to JSON - takes_value: false - - host: - short: h - required: false - long: host - value_name: host - help: Sets the remote host to connect to - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the remote port to connect to - takes_value: true - - runs: - short: r - required: false - long: runs - value_name: runs - takes_value: true - help: Sets the number of times the entire test should be run diff --git a/sky-bench/src/config.rs b/sky-bench/src/config.rs deleted file mode 100644 index f8e283a4..00000000 --- a/sky-bench/src/config.rs +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Created on Mon Aug 08 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::error::{BResult, Error}, - crate::util, - clap::ArgMatches, - std::{fmt::Display, str::FromStr}, -}; - -static mut OUTPUT_JSON: bool = false; - -#[derive(Clone)] -pub struct ServerConfig { - /// host - host: Box, - /// port - port: u16, - /// connection count for network pool - connections: usize, -} - -#[inline(always)] -fn try_update>(input: Option, target: &mut T) -> BResult<()> -where - ::Err: Display, -{ - if let Some(input) = input { - let parsed = input - .as_ref() - .parse::() - .map_err(|e| Error::Config(format!("parse error: `{}`", e)))?; - *target = parsed; - } - Ok(()) -} - -impl ServerConfig { - const DEFAULT_HOST: &'static str = "127.0.0.1"; - const DEFAULT_PORT: u16 = 2003; - const DEFAULT_CONNECTIONS: usize = 10; - /// Init the default server config - pub fn new(matches: &ArgMatches) -> BResult { - let mut slf = Self { - host: Self::DEFAULT_HOST.into(), - port: Self::DEFAULT_PORT, - connections: Self::DEFAULT_CONNECTIONS, - }; - slf.try_host(matches.value_of_lossy("host")); - slf.try_port(matches.value_of_lossy("port"))?; - slf.try_connections(matches.value_of_lossy("connections"))?; - Ok(slf) - } - /// Update the host - pub fn try_host>(&mut self, host: Option) { - if let Some(host) = host { - self.host = host.as_ref().into(); - } - } - /// Attempt to update the port - pub fn try_port>(&mut self, port: Option) -> BResult<()> { - try_update(port, &mut self.port) - } - /// Attempt to update the connections - pub fn try_connections>(&mut self, con: Option) -> BResult<()> { - try_update(con, &mut self.connections) - } -} - -impl ServerConfig { - pub fn host(&self) -> &str { - self.host.as_ref() - } - pub fn port(&self) -> u16 { - self.port - } - pub fn connections(&self) -> usize { - self.connections - } -} - -/// Benchmark configuration -#[derive(Clone)] -pub struct BenchmarkConfig { - pub server: ServerConfig, - kvsize: usize, - queries: usize, - runs: usize, -} - -impl BenchmarkConfig { - const DEFAULT_QUERIES: usize = 100_000; - const DEFAULT_KVSIZE: usize = 3; - const DEFAULT_RUNS: usize = 5; - pub fn new(server: &ServerConfig, matches: ArgMatches) -> BResult { - let mut slf = Self { - server: server.clone(), - queries: Self::DEFAULT_QUERIES, - kvsize: Self::DEFAULT_KVSIZE, - runs: Self::DEFAULT_RUNS, - }; - try_update(matches.value_of_lossy("queries"), &mut slf.queries)?; - try_update(matches.value_of_lossy("size"), &mut slf.kvsize)?; - try_update(matches.value_of_lossy("runs"), &mut slf.runs)?; - util::ensure_main_thread(); - unsafe { - OUTPUT_JSON = matches.is_present("json"); - } - Ok(slf) - } - pub fn kvsize(&self) -> usize { - self.kvsize - } - pub fn query_count(&self) -> usize { - self.queries - } - pub fn runs(&self) -> usize { - self.runs - } -} - -pub fn should_output_messages() -> bool { - util::ensure_main_thread(); - unsafe { !OUTPUT_JSON } -} diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs deleted file mode 100644 index c2519702..00000000 --- a/sky-bench/src/error.rs +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Created on Mon Aug 08 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - libstress::WorkpoolError, - skytable::error::Error as SkyError, - std::{collections::TryReserveError, fmt::Display}, -}; - -pub type BResult = Result; - -/// Benchmark tool errors -pub enum Error { - /// An error originating from the Skytable client - Client(SkyError), - /// An error originating from the benchmark/server configuration - Config(String), - /// A runtime error - Runtime(String), -} - -impl From for Error { - fn from(e: SkyError) -> Self { - Self::Client(e) - } -} - -impl Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Error::Client(e) => write!(f, "client error: {}", e), - Error::Config(e) => write!(f, "config error: {}", e), - Error::Runtime(e) => write!(f, "runtime error: {}", e), - } - } -} - -impl From for Error { - fn from(e: TryReserveError) -> Self { - Error::Runtime(format!("memory reserve error: {}", e)) - } -} - -impl From for Error { - fn from(e: WorkpoolError) -> Self { - Error::Runtime(format!("threadpool error: {}", e)) - } -} diff --git a/sky-bench/src/main.rs b/sky-bench/src/main.rs index 76ea1b2b..57a21b8e 100644 --- a/sky-bench/src/main.rs +++ b/sky-bench/src/main.rs @@ -1,5 +1,5 @@ /* - * Created on Mon Aug 08 2022 + * Created on Wed Nov 15 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -7,7 +7,7 @@ * vision to provide flexibility in data modelling without compromising * on performance, queryability or scalability. * - * Copyright (c) 2022, Sayan Nandan + * Copyright (c) 2023, Sayan Nandan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -24,40 +24,4 @@ * */ -use { - clap::{load_yaml, App}, - config::ServerConfig, - env_logger::Builder, - std::{env, process}, -}; -#[macro_use] -extern crate log; - -mod bench; -mod config; -mod error; -mod util; - -fn main() { - Builder::new() - .parse_filters(&env::var("SKYBENCH_LOG").unwrap_or_else(|_| "info".to_owned())) - .init(); - if let Err(e) = run() { - error!("sky-bench exited with error: {}", e); - process::exit(0x01); - } -} - -fn run() -> error::BResult<()> { - // init CLI arg parser - let cli_args = load_yaml!("cli.yml"); - let cli = App::from_yaml(cli_args); - let matches = cli.get_matches(); - - // parse args - let cfg = ServerConfig::new(&matches)?; - - // run our task - bench::run_bench(&cfg, matches)?; - util::cleanup(&cfg) -} +fn main() {} diff --git a/sky-bench/src/util.rs b/sky-bench/src/util.rs deleted file mode 100644 index fc2ab9bb..00000000 --- a/sky-bench/src/util.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Created on Tue Aug 09 2022 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2022, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::{ - config::ServerConfig, - error::{BResult, Error}, - }, - skytable::{Connection, Element, Query, RespCode}, - std::thread, -}; - -/// Check if the provided keysize has enough combinations to support the given `queries` count -/// -/// This function is heavily optimized and should take Θ(1) time. The `ALWAYS_TRUE_FACTOR` is -/// dependent on pointer width (more specifically the virtual address space size). -/// - For 64-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 12, we'll hit the maximum -/// of the address space and hence this will always return true (because of the size of `usize`) -/// > The value for r = 12 is `1.27309515e+20` which largely exceeds `1.8446744e+19` -/// - For 32-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 5, we'll hit the maximum -/// of the address space and hence this will always return true (because of the size of `usize`) -/// > The value for r = 5 is `8.81e+9` which largely exceeds `4.3e+9` -pub const fn has_enough_ncr(keysize: usize, queries: usize) -> bool { - const LUT: [u64; 11] = [ - // 1B - 256, - // 2B - 32640, - // 3B - 2763520, - // 4B - 174792640, - // 5B - 8809549056, - // 6B - 368532802176, - // 7B - 13161885792000, - // 8B - 409663695276000, - // 9B - 11288510714272000, - // 10B - 278826214642518400, - // 11B - 6235568072914502400, - ]; - #[cfg(target_pointer_width = "64")] - const ALWAYS_TRUE_FACTOR: usize = 12; - #[cfg(target_pointer_width = "32")] - const ALWAYS_TRUE_FACTOR: usize = 5; - keysize >= ALWAYS_TRUE_FACTOR || (LUT[keysize - 1] >= queries as _) -} - -/// Run a sanity test, making sure that the server is ready for benchmarking. This function will do the -/// following tests: -/// - Connect to the instance -/// - Run a `heya` as a preliminary test -/// - Create a new table `tmpbench`. This is where we're supposed to run all the benchmarks. -/// - Switch to the new table -/// - Set a key, and get it checking the equality of the returned value -pub fn run_sanity_test(server_config: &ServerConfig) -> BResult<()> { - let mut con = Connection::new(server_config.host(), server_config.port())?; - let tests: [(Query, Element, &str); 5] = [ - ( - Query::from("HEYA"), - Element::String("HEY!".to_owned()), - "heya", - ), - ( - Query::from("CREATE MODEL default.tmpbench(binary, binary)"), - Element::RespCode(RespCode::Okay), - "create model", - ), - ( - Query::from("use default.tmpbench"), - Element::RespCode(RespCode::Okay), - "use", - ), - ( - Query::from("set").arg("x").arg("100"), - Element::RespCode(RespCode::Okay), - "set", - ), - ( - Query::from("get").arg("x"), - Element::Binstr("100".as_bytes().to_owned()), - "get", - ), - ]; - for (query, expected, test_kind) in tests { - let r: Element = con.run_query(query)?; - if r != expected { - return Err(Error::Runtime(format!( - "sanity test for `{test_kind}` failed" - ))); - } - } - Ok(()) -} - -/// Ensures that the current thread is the main thread. If not, this function will panic -pub fn ensure_main_thread() { - assert_eq!( - thread::current().name().unwrap(), - "main", - "unsafe function called from non-main thread" - ) -} - -/// Run a cleanup. This function attempts to remove the `default.tmpbench` entity -pub fn cleanup(server_config: &ServerConfig) -> BResult<()> { - let mut c = Connection::new(server_config.host(), server_config.port())?; - let r: Element = c.run_query(Query::from("drop model default.tmpbench force"))?; - if r == Element::RespCode(RespCode::Okay) { - Err(Error::Runtime("failed to run cleanup".into())) - } else { - Ok(()) - } -} diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml deleted file mode 100644 index 30dfd28f..00000000 --- a/sky-migrate/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "sky-migrate" -version = "0.8.0" -authors = ["Sayan Nandan "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -skytable = { git = "https://github.com/skytable/client-rust.git" } -env_logger = "0.10.0" -bincode = "1.3.3" -log = "0.4.19" -clap = { version = "2", features = ["yaml"] } diff --git a/sky-migrate/README.md b/sky-migrate/README.md deleted file mode 100644 index 58b75930..00000000 --- a/sky-migrate/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Skytable migration tool - -The Skytable migration tool can be used to perform migrations between database versions. The basic idea is: -"read all data from the older machine and send it over to the newer one." - -For migrating versions from 0.6 to 0.7, the database administrator has to launch the tool in the old data directory of the old instance and then pass the new instance host/port information. The tool will then read the data from the directory (this was possible because 0.6 used a very simple disk format than newer versions). This approach however has the advantage of not having to start the database server for the migration to happen. - -## License - -All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/sky-migrate/src/cli.yml b/sky-migrate/src/cli.yml deleted file mode 100644 index 97bde94d..00000000 --- a/sky-migrate/src/cli.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Skytable Migration Tool -version: 0.7.0 -author: Sayan N. -about: | - The Skytable migration tool allows users coming from older versions (>=0.8.0) - to upgrade their datasets to the latest Skytable version. This tool currently - supports versions >= 0.8.0 and upgrading it to 0.7.0. To upgrade, on needs - to simply run: - sky-migrate --prevdir --new : - Where `` is the path to the last installation's data directory and - `` and `` is the hostname and port for the new server instance -args: - - new: - long: new - takes_value: true - required: true - help: The : combo for the new instance - value_name: new - - prevdir: - long: prevdir - takes_value: true - required: true - help: Path to the previous installation location - value_name: prevdir - - serial: - long: serial - takes_value: false - required: false - help: | - Transfer entries one-by-one instead of all at once to save memory diff --git a/sky-migrate/src/main.rs b/sky-migrate/src/main.rs deleted file mode 100644 index f77bcf6a..00000000 --- a/sky-migrate/src/main.rs +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Created on Tue Aug 17 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![allow(clippy::unit_arg)] - -use { - clap::{load_yaml, App}, - core::hint::unreachable_unchecked, - env_logger::Builder, - log::{error as err, info}, - skytable::{query, sync::Connection, Element, Query, RespCode}, - std::{collections::HashMap, env, fs, path::PathBuf, process}, -}; - -type Bytes = Vec; - -fn main() { - // first evaluate config - let cfg_layout = load_yaml!("cli.yml"); - let matches = App::from_yaml(cfg_layout).get_matches(); - Builder::new() - .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned())) - .init(); - let new_host = matches - .value_of("new") - .map(|v| v.to_string()) - .unwrap_or_else(|| unsafe { unreachable_unchecked() }); - let serial = matches.is_present("serial"); - let hostsplit: Vec<&str> = new_host.split(':').collect(); - if hostsplit.len() != 2 { - err(err!("Bad value for --new")); - } - let (host, port) = unsafe { (hostsplit.get_unchecked(0), hostsplit.get_unchecked(1)) }; - let port = match port.parse() { - Ok(p) => p, - Err(e) => err(err!("Bad value for port in --new: {}", e)), - }; - let mut old_dir = matches - .value_of("prevdir") - .map(PathBuf::from) - .unwrap_or_else(|| unsafe { unreachable_unchecked() }); - old_dir.push("data.bin"); - // now connect - let mut con = match Connection::new(host, port) { - Ok(con) => con, - Err(e) => err(err!("Failed to connect to new instance with error: {}", e)), - }; - // run sanity test - let q = query!("HEYA"); - match con.run_query_raw(&q) { - Ok(Element::String(s)) if s.eq("HEY!") => {} - Ok(_) => err(err!("Unknown response from server")), - Err(e) => err(err!( - "An I/O error occurred while running sanity test: {}", - e - )), - } - info!("Sanity test complete"); - - // now de old file - let read = match fs::read(old_dir) { - Ok(r) => r, - Err(e) => err(err!( - "Failed to read data.bin file from old directory: {}", - e - )), - }; - let de: HashMap = match bincode::deserialize(&read) { - Ok(r) => r, - Err(e) => err(err!("Failed to unpack old file with: {}", e)), - }; - unsafe { - if serial { - // transfer serially - for (key, value) in de.into_iter() { - let q = query!( - "USET", - String::from_utf8_unchecked(key), - String::from_utf8_unchecked(value) - ); - okay(&mut con, q) - } - } else { - // transfer all at once - let mut query = Query::from("USET"); - for (key, value) in de.into_iter() { - query.push(String::from_utf8_unchecked(key)); - query.push(String::from_utf8_unchecked(value)); - } - okay(&mut con, query) - } - } - info!("Finished migration"); -} - -fn err(_i: ()) -> ! { - process::exit(0x01) -} - -fn okay(con: &mut Connection, q: Query) { - match con.run_query_raw(&q) { - Ok(Element::RespCode(RespCode::Okay)) => {} - Err(e) => err(err!("An I/O error occurred while running query: {}", e)), - Ok(_) => err(err!("Unknown response from server")), - } -} diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml deleted file mode 100644 index f05bb998..00000000 --- a/stress-test/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "stress-test" -version = "0.1.0" -authors = ["Sayan Nandan "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -# internal deps -libstress = { path = "../libstress" } -skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next", features = [ - "dbg", -] } -devtimer = "4.0.1" -# external deps -sysinfo = "0.29.7" -env_logger = "0.10.0" -log = "0.4.19" -rand = "0.8.5" -crossbeam-channel = "0.5.8" diff --git a/stress-test/README.md b/stress-test/README.md deleted file mode 100644 index 5008306b..00000000 --- a/stress-test/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Skytable stress-test tool - -This is a tool in its infancy, but its ultimate goal is to provide a stress testing framework for Skytable. For now, it tests linearity (core scalability) with increasing clients. - -## License - -All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/stress-test/src/linearity_client.rs b/stress-test/src/linearity_client.rs deleted file mode 100644 index d97e37b7..00000000 --- a/stress-test/src/linearity_client.rs +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Created on Fri Jun 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -//! # Client linearity tests -//! -//! This module contains functions to test the linearity of the database with increasing number -//! of clients, i.e how the number of queries scale with increasing clients. These functions -//! however, DO NOT focus on benchmarking and instead focus on correctness under load from -//! concurrent clients. -//! - -use { - crate::{logstress, DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV}, - crossbeam_channel::bounded, - devtimer::SimpleTimer, - libstress::{rayon::prelude::*, utils::generate_random_string_vector, Workpool}, - skytable::{actions::Actions, query, Connection, Element, Query, RespCode}, -}; - -macro_rules! log_client_linearity { - ($stressid:expr, $counter:expr, $what:expr) => { - log::info!( - "Stress ({}{}) [{}]: Clients: {}; K/V size: {}; Queries: {}", - $stressid, - $counter, - $what, - $counter, - DEFAULT_SIZE_KV, - DEFAULT_QUERY_COUNT - ); - }; -} - -/// This object provides methods to measure the percentage change in the slope -/// of a function that is expected to have linearity. -/// -/// For example, we can think of it to work in the following way: -/// Let h(x) be a function with linearity (not proportionality because that isn't -/// applicable in our case). h(x) gives us the time taken to run a given number of -/// queries (invariant; not plotted on axes), where x is the number of concurrent -/// clients. As we would want our database to scale with increasing clients (and cores), -/// we'd expect linearity, hence the gradient should continue to fall with increasing -/// values in the +ve x-axis effectively producing a constantly decreasing slope, reflected -/// by increasing values of abs(get_delta(h(x))). -/// -/// TODO(@ohsayan): Of course, some unexpected kernel errors/scheduler hiccups et al can -/// cause there to be a certain epsilon that must be tolerated with a tolerance factor -/// -pub struct LinearityMeter { - init: Option, - measure: Vec, -} - -impl LinearityMeter { - pub const fn new() -> Self { - Self { - init: None, - measure: Vec::new(), - } - } - pub fn get_delta(&mut self, current: u128) -> f32 { - if let Some(u) = self.init { - let cur = ((current as f32 - u as f32) / u as f32) * 100.00_f32; - self.measure.push(cur); - cur - } else { - // if init is not initialized, initialize it - self.init = Some(current); - // no change when at base - 0.00 - } - } -} - -pub fn stress_linearity_concurrent_clients_set( - mut rng: &mut impl rand::Rng, - max_workers: usize, - temp_con: &mut Connection, -) { - logstress!( - "A [SET]", - "Linearity test with monotonically increasing clients" - ); - let mut current_thread_count = 1usize; - - // generate the random k/v pairs - let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true); - let values = - generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false); - let (keys, values) = match (keys, values) { - (Ok(k), Ok(v)) => (k, v), - _ => { - eprintln!("Allocation error"); - std::process::exit(0x01); - } - }; - // make sure the database is empty - temp_con.flushdb().unwrap(); - - // initialize the linearity counter - let mut linearity = LinearityMeter::new(); - while current_thread_count <= max_workers { - log_client_linearity!("A", current_thread_count, "SET"); - // generate the set packets - let set_packs: Vec = keys - .par_iter() - .zip(values.par_iter()) - .map(|(k, v)| query!("SET", k, v)) - .collect(); - let workpool = Workpool::new( - current_thread_count, - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - assert_eq!( - sock.run_query_raw(&query).unwrap(), - Element::RespCode(RespCode::Okay) - ); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - let mut timer = SimpleTimer::new(); - timer.start(); - workpool.execute_and_finish_iter(set_packs); - timer.stop(); - log::info!( - "Delta: {}%", - linearity.get_delta(timer.time_in_nanos().unwrap()) - ); - // clean up the database - temp_con.flushdb().unwrap(); - current_thread_count += 1; - } -} - -pub fn stress_linearity_concurrent_clients_get( - mut rng: &mut impl rand::Rng, - max_workers: usize, - temp_con: &mut Connection, -) { - logstress!( - "A [GET]", - "Linearity test with monotonically increasing clients" - ); - let mut current_thread_count = 1usize; - - // generate the random k/v pairs - let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true); - let values = - generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false); - let (keys, values) = match (keys, values) { - (Ok(k), Ok(v)) => (k, v), - _ => { - eprintln!("Allocation error"); - std::process::exit(0x01); - } - }; - - // Make sure that the database is empty - temp_con.flushdb().unwrap(); - - // First set the keys - let set_packs: Vec = keys - .par_iter() - .zip(values.par_iter()) - .map(|(k, v)| query!("SET", k, v)) - .collect(); - let workpool = Workpool::new_default_threads( - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - assert_eq!( - sock.run_query_raw(&query).unwrap(), - Element::RespCode(RespCode::Okay) - ); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - workpool.execute_and_finish_iter(set_packs); - - // initialize the linearity counter - let mut linearity = LinearityMeter::new(); - while current_thread_count <= max_workers { - log_client_linearity!("A", current_thread_count, "GET"); - /* - We create a mpmc to receive the results returned. This avoids us using - any kind of locking on the surface which can slow down things - */ - let (tx, rx) = bounded::(DEFAULT_QUERY_COUNT); - - // generate the get packets - let get_packs: Vec = keys.iter().map(|k| query!("GET", k)).collect(); - let wp = Workpool::new( - current_thread_count, - || Connection::new("127.0.0.1", 2003).unwrap(), - move |sock, query| { - let tx = tx.clone(); - tx.send(sock.run_query_raw(&query).unwrap()).unwrap(); - }, - |_| {}, - true, - Some(DEFAULT_QUERY_COUNT), - ) - .unwrap(); - let mut timer = SimpleTimer::new(); - timer.start(); - wp.execute_and_finish_iter(get_packs); - timer.stop(); - log::info!( - "Delta: {}%", - linearity.get_delta(timer.time_in_nanos().unwrap()) - ); - let rets: Vec = rx - .into_iter() - .map(|v| { - if let Element::String(val) = v { - val - } else { - panic!("Unexpected response from server"); - } - }) - .collect(); - assert_eq!( - rets.len(), - values.len(), - "Incorrect number of values returned by server" - ); - - // now evaluate them - assert!( - rets.into_par_iter().all(|v| values.contains(&v)), - "Values returned by the server don't match what was sent" - ); - current_thread_count += 1; - } - temp_con.flushdb().unwrap(); -} diff --git a/stress-test/src/main.rs b/stress-test/src/main.rs deleted file mode 100644 index e59045ef..00000000 --- a/stress-test/src/main.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Created on Wed Jun 16 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#![deny(unused_crate_dependencies)] -#![deny(unused_imports)] - -use std::thread::available_parallelism; - -use { - libstress::traits::ExitError, - log::{info, trace, warn}, - rand::thread_rng, - skytable::Connection, - std::env, - sysinfo::{RefreshKind, System, SystemExt}, -}; -mod linearity_client; -mod utils; - -pub const DEFAULT_SIZE_KV: usize = 4; -pub const DEFAULT_QUERY_COUNT: usize = 100_000_usize; - -#[macro_export] -macro_rules! logstress { - ($stressid:expr, $extra:expr) => { - log::info!("Stress ({}): {}", $stressid, $extra); - }; -} - -fn main() { - // Build the logger - env_logger::Builder::new() - .parse_filters(&env::var("SKY_STRESS_LOG").unwrap_or_else(|_| "trace".to_owned())) - .init(); - warn!("The stress test checks correctness under load and DOES NOT show the true throughput"); - - // get the rng and refresh sysinfo - let mut rng = thread_rng(); - // we only need to refresh memory and CPU info; don't waste time syncing other things - let to_refresh = RefreshKind::new().with_memory(); - let mut sys = System::new_with_specifics(to_refresh); - sys.refresh_specifics(to_refresh); - let core_count = available_parallelism().map_or(1, usize::from); - let max_workers = core_count * 2; - trace!( - "This host has {} logical cores. Will spawn a maximum of {} threads", - core_count, - max_workers * 2 - ); - - // establish a connection to ensure sanity - let mut temp_con = Connection::new("127.0.0.1", 2003).exit_error("Failed to connect to server"); - - // calculate the maximum keylen - let max_keylen = utils::calculate_max_keylen(DEFAULT_QUERY_COUNT, &mut sys); - info!( - "This host can support a maximum theoretical keylen of: {}", - max_keylen - ); - - // run the actual stress tests - linearity_client::stress_linearity_concurrent_clients_set(&mut rng, max_workers, &mut temp_con); - linearity_client::stress_linearity_concurrent_clients_get(&mut rng, max_workers, &mut temp_con); - - // done, exit - info!("SUCCESS. Stress test complete!"); -} diff --git a/stress-test/src/utils.rs b/stress-test/src/utils.rs deleted file mode 100644 index 76cbbc8f..00000000 --- a/stress-test/src/utils.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Created on Fri Jun 18 2021 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2021, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ -use { - log::trace, - skytable::Query, - sysinfo::{System, SystemExt}, -}; - -pub fn calculate_max_keylen(expected_queries: usize, sys: &mut System) -> usize { - let total_mem_in_bytes = (sys.total_memory() * 1024) as usize; - trace!( - "This host has a total memory of: {} Bytes", - total_mem_in_bytes - ); - // av_mem gives us 90% of the memory size - let ninety_percent_of_memory = (0.90_f32 * total_mem_in_bytes as f32) as usize; - let mut highest_len = 1usize; - loop { - let set_pack_len = Query::array_packet_size_hint(vec![3, highest_len, highest_len]); - let get_pack_len = Query::array_packet_size_hint(vec![3, highest_len]); - let resulting_size = expected_queries - * ( - // for the set packets - set_pack_len + - // for the get packets - get_pack_len + - // for the keys themselves - highest_len - ); - if resulting_size >= ninety_percent_of_memory as usize { - break; - } - // increase the length by 5% every time to get the maximum possible length - // now this 5% increment is a tradeoff, but it's worth it to not wait for - // so long - highest_len = (highest_len as f32 * 1.05_f32).ceil() as usize; - } - highest_len -}