From 36b3fc0ee8227d0f7135a35782678f8c7f2ef6b0 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Tue, 31 May 2022 01:10:58 -0700 Subject: [PATCH] Add docs for binary crates and upgrade deps --- Cargo.lock | 25 +++++++++++-------------- README.md | 2 +- cli/README.md | 9 +++++++++ harness/README.md | 7 +++++++ libstress/Cargo.toml | 1 - libstress/src/lib.rs | 2 +- server/Cargo.toml | 2 +- server/README.md | 7 +++++++ sky-bench/README.md | 20 ++++++++++++++++++++ sky-migrate/README.md | 10 ++++++++++ stress-test/README.md | 7 +++++++ 11 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 cli/README.md create mode 100644 harness/README.md create mode 100644 server/README.md create mode 100644 sky-bench/README.md create mode 100644 sky-migrate/README.md create mode 100644 stress-test/README.md diff --git a/Cargo.lock b/Cargo.lock index 86462d0a..c409cf76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,7 +98,7 @@ dependencies = [ "async-trait", "futures-channel", "futures-util", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "tokio", ] @@ -328,7 +328,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", "winapi", @@ -460,13 +460,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if", "crc32fast", - "libc", "miniz_oxide", ] @@ -675,7 +673,6 @@ version = "0.8.0" dependencies = [ "crossbeam-channel", "log", - "num_cpus", "rand", "rayon", ] @@ -722,9 +719,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] @@ -884,9 +881,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core 0.9.3", @@ -1163,7 +1160,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" dependencies = [ - "parking_lot 0.12.0", + "parking_lot 0.12.1", ] [[package]] @@ -1308,7 +1305,7 @@ dependencies = [ "libstress", "log", "openssl", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "rand", "rcrypt", "regex", @@ -1515,7 +1512,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/README.md b/README.md index bd8bccdb..ee83688b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ You can learn more about installation [here](https://docs.skytable.io/getting-st ## Features -- **Insanely fast**: Scale to millions of queries per second per node +- **Insanely fast**: Scale to millions of queries per second per node. See [benchmarks here](https://github.com/ohsayan/sky-benches). - **Multiple keyspaces/tables**: Seamlessly integrates with actions to provide a SQL-like experience - **Key/value store**: `GET` , `SET` , `UPDATE` and [all that stuff](https://docs.skytable.io/all-actions). With the `str` and `binstr` types. - **Authn/Authz**: Simple and secure authentication/authorization diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 00000000..e17064ee --- /dev/null +++ b/cli/README.md @@ -0,0 +1,9 @@ +# Skytable Shell + +This directory contains the source code for the Skytable Shell. When compiled, it produces a single binary `skysh`, short for **Sky**table **s**hell. The Skytable shell can be used to connect to a Skytable instance and it provides some nice to have shell features like history, terminal interactions and more. + +By default, `skysh` connects to the `default:default` entity for an instance running on `127.0.0.1` (localhost) on port `2003`. You can change these connection settings use command-line parameters. + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/harness/README.md b/harness/README.md new file mode 100644 index 00000000..257b3fd7 --- /dev/null +++ b/harness/README.md @@ -0,0 +1,7 @@ +# Harness + +Harness is the custom build tool that is used by Skytable as a "test harness" while also providing bundling/packaging functions. The `Makefile` at the root of the source tree builds the `harness` binary and then runs respective commands using the harness binary (such as `test`, `build`, ...). + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/libstress/Cargo.toml b/libstress/Cargo.toml index c4c362c5..65c45661 100644 --- a/libstress/Cargo.toml +++ b/libstress/Cargo.toml @@ -8,7 +8,6 @@ edition = "2021" [dependencies] # external deps -num_cpus = "1.13.1" crossbeam-channel = "0.5.4" rayon = "1.5.3" log = "0.4.17" diff --git a/libstress/src/lib.rs b/libstress/src/lib.rs index 8e5bd9ec..22bee825 100644 --- a/libstress/src/lib.rs +++ b/libstress/src/lib.rs @@ -336,7 +336,7 @@ where ) -> Self { // we'll naively use the number of CPUs present on the system times 2 to determine // the number of workers (sure the scheduler does tricks all the time) - let worker_count = num_cpus::get() * 2; + let worker_count = thread::available_parallelism().map_or(1, usize::from) * 2; Self::new( worker_count, init_pre_loop_var, diff --git a/server/Cargo.toml b/server/Cargo.toml index 6af2827f..48a858ef 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,7 +18,7 @@ env_logger = "0.9.0" hashbrown = { version = "0.12.1", features = ["raw"] } log = "0.4.17" openssl = { version = "0.10.40", features = ["vendored"] } -parking_lot = "0.12.0" +parking_lot = "0.12.1" regex = "1.5.6" serde = { version = "1.0.137", features = ["derive"] } tokio = { version = "1.18.2", features = ["full"] } diff --git a/server/README.md b/server/README.md new file mode 100644 index 00000000..2b3b775b --- /dev/null +++ b/server/README.md @@ -0,0 +1,7 @@ +# Skytable + +This directory contains the source code for the Skytable database server (the "actual database"). When compiled, it produces a single binary `skyd`, short for **Sky**table **d**aemon. The daemon serves the clients over a network. + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/sky-bench/README.md b/sky-bench/README.md new file mode 100644 index 00000000..69869217 --- /dev/null +++ b/sky-bench/README.md @@ -0,0 +1,20 @@ +# Skytable Benchmark Tool + +`sky-bench` is Skytable's benchmarking tool. Unlike most other benchmarking tools, Skytable's benchmark +tool doesn't do anything "fancy" to make benchmarks appear better than they are. As it happens, the benchmark tool might show Skytable to be slower! + +Here's how the benchmark tool works (it's dead simple): + +1. Depending on the configuration it launches "network pools" which are just thread pools where each worker + holds a persistent connection to the database (something like a connection pool) +2. A collection of unique, random keys are generated using a PRNG provided by the `rand` library that is + seeded using the OS' source of randomness. The values are allowed to repeat +3. The [Skytable Rust driver](https://github.com/skytable/client-rust) is used to generate _raw query packets_. To put it simply, the keys and values are turned into `Query` objects and then into the raw bytes that will be sent over the network. (This just makes it simpler to design the network pool interface) +4. For every type of benchmark (GET,SET,...) we use the network pool to send all the bytes and wait until we receive the expected response. We time how long it takes to send and receive the response for all the queries for the given test (aggregate) +5. We repeat this for all the remaining tests +6. We repeat the entire set of tests 5 times (by default, this can be changed). +7. We do the calculations and output the results. + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/sky-migrate/README.md b/sky-migrate/README.md new file mode 100644 index 00000000..58b75930 --- /dev/null +++ b/sky-migrate/README.md @@ -0,0 +1,10 @@ +# Skytable migration tool + +The Skytable migration tool can be used to perform migrations between database versions. The basic idea is: +"read all data from the older machine and send it over to the newer one." + +For migrating versions from 0.6 to 0.7, the database administrator has to launch the tool in the old data directory of the old instance and then pass the new instance host/port information. The tool will then read the data from the directory (this was possible because 0.6 used a very simple disk format than newer versions). This approach however has the advantage of not having to start the database server for the migration to happen. + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE). diff --git a/stress-test/README.md b/stress-test/README.md new file mode 100644 index 00000000..5008306b --- /dev/null +++ b/stress-test/README.md @@ -0,0 +1,7 @@ +# Skytable stress-test tool + +This is a tool in its infancy, but its ultimate goal is to provide a stress testing framework for Skytable. For now, it tests linearity (core scalability) with increasing clients. + +## License + +All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).