From 0087d6b07c0191fc6336185ed132161bdfce50a5 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Fri, 18 Dec 2020 16:14:23 +0530 Subject: [PATCH 01/17] Add basic SSL/TLS listener This commit adds a basic SSL/TLS listener using `openssl`. The `SslListener` object can accept a connection and get a decrypted stream. Signed-off-by: Sayan Nandan --- Cargo.lock | 91 ++++++++++++++++++++++++++++++++++-- server/Cargo.toml | 4 +- server/src/dbnet.rs | 3 +- server/src/main.rs | 1 + server/src/tls/mod.rs | 104 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 server/src/tls/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 576d4841..5fae28fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chrono" version = "0.4.19" @@ -138,6 +144,21 @@ dependencies = [ "termcolor", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fs_extra" version = "1.2.0" @@ -156,7 +177,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -245,7 +266,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -321,6 +342,43 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +[[package]] +name = "openssl" +version = "0.10.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d008f51b1acffa0d3450a68606e6a51c123012edaacb0f4e1426bd978869187" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-src" +version = "111.13.0+1.1.1i" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045e4dc48af57aad93d665885789b43222ae26f4886494da12d1ed58d309dcb6" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de52d8eabd217311538a39bba130d7dea1f1e118010fee7a033d966845e7d5fe" +dependencies = [ + "autocfg", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.1" @@ -338,7 +396,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi", "instant", "libc", @@ -353,6 +411,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "ppv-lite86" version = "0.2.9" @@ -513,7 +577,7 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "redox_syscall", "winapi", @@ -549,12 +613,15 @@ dependencies = [ "lazy_static", "libtdb", "log", + "openssl", + "openssl-sys", "parking_lot", "regex", "serde", "serde_derive", "tdb_macros", "tokio", + "tokio-openssl-vendored", "toml", ] @@ -651,6 +718,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-openssl-vendored" +version = "0.5.0" +source = "git+https://github.com/terrabasedb/tokio-openssl-vendored.git?branch=next#048622804dd641428b51a0bcc72cd9c801d825e7" +dependencies = [ + "openssl", + "openssl-sys", + "tokio", +] + [[package]] name = "toml" version = "0.5.7" @@ -684,6 +761,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/server/Cargo.toml b/server/Cargo.toml index 778a7f68..80334990 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -22,7 +22,9 @@ log = "0.4.11" chrono = "0.4.19" regex = "1.4.2" tdb_macros = {path="../tdb-macros"} - +tokio-openssl-vendored = {git="https://github.com/terrabasedb/tokio-openssl-vendored.git", branch="next"} +openssl = "0.10.31" +openssl-sys = "0.9.59" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.3.2" diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index 85a9598b..3e81816b 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -52,7 +52,6 @@ use tokio::net::TcpStream; use tokio::sync::Semaphore; use tokio::sync::{broadcast, mpsc}; use tokio::time::{self, Duration}; - /// Responsible for gracefully shutting down the server instead of dying randomly // Sounds very sci-fi ;) pub struct Terminator { @@ -104,7 +103,7 @@ pub struct Listener { } /// A per-connection handler -struct CHandler { +pub struct CHandler { db: CoreDB, con: Connection, climit: Arc, diff --git a/server/src/main.rs b/server/src/main.rs index 9389552a..75334b46 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -38,6 +38,7 @@ mod kvengine; mod protocol; mod queryengine; mod resp; +mod tls; use coredb::CoreDB; use dbnet::run; use env_logger::*; diff --git a/server/src/tls/mod.rs b/server/src/tls/mod.rs new file mode 100644 index 00000000..02e73c72 --- /dev/null +++ b/server/src/tls/mod.rs @@ -0,0 +1,104 @@ +/* + * Created on Fri Dec 18 2020 + * + * This file is a part of TerrabaseDB + * Copyright (c) 2020, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#[allow(dead_code)] // TODO: Don't keep clippy quiet! +// use crate::dbnet::CHandler; +// use crate::dbnet::Terminator; +// use crate::protocol::Connection; +use crate::CoreDB; +use libtdb::TResult; +use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod}; +// use std::net::ToSocketAddrs; +use std::sync::Arc; +// use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::sync::Semaphore; +use tokio::sync::{broadcast, mpsc}; +use tokio::time::{self, Duration}; +use tokio_openssl_vendored::SslStream; +pub struct SslListener { + /// An atomic reference to the coretable + db: CoreDB, + /// The incoming connection listener (binding) + listener: TcpListener, + /// The maximum number of connections + climit: Arc, + /// The shutdown broadcaster + signal: broadcast::Sender<()>, + // When all `Sender`s are dropped - the `Receiver` gets a `None` value + // We send a clone of `terminate_tx` to each `CHandler` + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + acceptor: Arc, +} + +impl SslListener { + pub fn new_pem_based_ssl_connection( + key_file: String, + chain_file: String, + db: CoreDB, + listener: TcpListener, + climit: Arc, + signal: broadcast::Sender<()>, + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + ) -> TResult { + let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?; + acceptor.set_private_key_file(key_file, SslFiletype::PEM)?; + acceptor.set_certificate_chain_file(chain_file)?; + let acceptor = Arc::new(acceptor.build()); + Ok(SslListener { + db, + listener, + climit, + signal, + terminate_tx, + terminate_rx, + acceptor, + }) + } + async fn accept(&mut self) -> TResult> { + let mut backoff = 1; + loop { + match self.listener.accept().await { + // We don't need the bindaddr + // We get the encrypted stream which we need to decrypt + // by using the acceptor + Ok((encrypted_stream, _)) => { + let decrypted_stream = + tokio_openssl_vendored::accept(&self.acceptor, encrypted_stream).await?; + return Ok(decrypted_stream); + } + Err(e) => { + if backoff > 64 { + // Too many retries, goodbye user + return Err(e.into()); + } + } + } + // Wait for the `backoff` duration + time::sleep(Duration::from_secs(backoff)).await; + // We're using exponential backoff + backoff *= 2; + } + } +} From 43bb1789545c01840dc00d72742bba6c307c1f5e Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Wed, 23 Dec 2020 14:08:42 +0530 Subject: [PATCH 02/17] Implement basic TLS module and connection handler This commit implements 'TLS versions' of `CHandler` and `Listener` from `dbnet`. Signed-off-by: Sayan Nandan --- .idea/.gitignore | 8 + Cargo.lock | 7 +- server/Cargo.toml | 4 +- server/src/coredb/mod.rs | 19 ++- server/src/dbnet.rs | 2 +- server/src/kvengine/dbsize.rs | 3 +- server/src/kvengine/del.rs | 3 +- server/src/kvengine/exists.rs | 3 +- server/src/kvengine/flushdb.rs | 3 +- server/src/kvengine/get.rs | 3 +- server/src/kvengine/jget.rs | 3 +- server/src/kvengine/keylen.rs | 3 +- server/src/kvengine/mget.rs | 3 +- server/src/kvengine/mod.rs | 3 +- server/src/kvengine/mset.rs | 3 +- server/src/kvengine/mupdate.rs | 3 +- server/src/kvengine/set.rs | 3 +- server/src/kvengine/strong.rs | 3 +- server/src/kvengine/update.rs | 3 +- server/src/kvengine/uset.rs | 3 +- server/src/main.rs | 1 - server/src/protocol/mod.rs | 1 + server/src/protocol/tls.rs | 274 +++++++++++++++++++++++++++++++++ server/src/resp/mod.rs | 125 +++++++++------ server/src/tls/mod.rs | 104 ------------- 25 files changed, 408 insertions(+), 182 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 server/src/protocol/tls.rs delete mode 100644 server/src/tls/mod.rs diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..73f69e09 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/Cargo.lock b/Cargo.lock index 5fae28fa..7f1f8d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -621,7 +621,7 @@ dependencies = [ "serde_derive", "tdb_macros", "tokio", - "tokio-openssl-vendored", + "tokio-openssl", "toml", ] @@ -719,9 +719,10 @@ dependencies = [ ] [[package]] -name = "tokio-openssl-vendored" +name = "tokio-openssl" version = "0.5.0" -source = "git+https://github.com/terrabasedb/tokio-openssl-vendored.git?branch=next#048622804dd641428b51a0bcc72cd9c801d825e7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01e5cc2d3a154fa310982d0affaec8140d6476805422265b2f648eb813f937f" dependencies = [ "openssl", "openssl-sys", diff --git a/server/Cargo.toml b/server/Cargo.toml index 80334990..7efd6911 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -22,8 +22,8 @@ log = "0.4.11" chrono = "0.4.19" regex = "1.4.2" tdb_macros = {path="../tdb-macros"} -tokio-openssl-vendored = {git="https://github.com/terrabasedb/tokio-openssl-vendored.git", branch="next"} -openssl = "0.10.31" +tokio-openssl = "0.5.0" +openssl = { version = "0.10", features = ["vendored"] } openssl-sys = "0.9.59" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.3.2" diff --git a/server/src/coredb/mod.rs b/server/src/coredb/mod.rs index 42b50e95..14051aa2 100644 --- a/server/src/coredb/mod.rs +++ b/server/src/coredb/mod.rs @@ -25,8 +25,8 @@ use crate::config::BGSave; use crate::config::SnapshotConfig; use crate::config::SnapshotPref; use crate::diskstore; -use crate::protocol::Connection; -use crate::protocol::Query; +use crate::protocol::tls::SslConnection; +use crate::protocol::{Connection, Query}; use crate::queryengine; use bytes::Bytes; use diskstore::PERSIST_FILE; @@ -245,14 +245,21 @@ impl CoreDB { } /// Execute a query that has already been validated by `Connection::read_query` - pub async fn execute_query(&self, query: Query, con: &mut Connection) -> TResult<()> { + pub async fn execute_query(&self, query: Query, mut con: &mut Connection) -> TResult<()> { match query { - Query::Simple(q) => queryengine::execute_simple(&self, con, q).await?, + Query::Simple(q) => { + queryengine::execute_simple(&self, &mut con, q).await?; + // Once we're done executing, flush the stream + con.flush_stream().await + } // TODO(@ohsayan): Pipeline commands haven't been implemented yet Query::Pipelined(_) => unimplemented!(), } - // Once we're done executing, flush the stream - con.flush_stream().await + } + + pub async fn execute_query_ssl(&self, _query: Query, _con: &mut SslConnection) -> TResult<()> { + // TODO(@ohsayan): Implement SSL-connection queries + todo!() } /// Create a new `CoreDB` instance diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index 3e81816b..aaf70309 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -158,7 +158,7 @@ impl Listener { impl CHandler { /// Process the incoming connection - async fn run(&mut self) -> TResult<()> { + pub async fn run(&mut self) -> TResult<()> { while !self.terminator.is_termination_signal() { let try_df = tokio::select! { tdf = self.con.read_query() => tdf, diff --git a/server/src/kvengine/dbsize.rs b/server/src/kvengine/dbsize.rs index 1144582d..192392f6 100644 --- a/server/src/kvengine/dbsize.rs +++ b/server/src/kvengine/dbsize.rs @@ -19,7 +19,8 @@ * */ use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; diff --git a/server/src/kvengine/del.rs b/server/src/kvengine/del.rs index 7c924634..6437ac8c 100644 --- a/server/src/kvengine/del.rs +++ b/server/src/kvengine/del.rs @@ -23,7 +23,8 @@ //! This module provides functions to work with `DEL` queries use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; diff --git a/server/src/kvengine/exists.rs b/server/src/kvengine/exists.rs index ee724b26..9ffe055a 100644 --- a/server/src/kvengine/exists.rs +++ b/server/src/kvengine/exists.rs @@ -23,7 +23,8 @@ //! This module provides functions to work with `EXISTS` queries use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; diff --git a/server/src/kvengine/flushdb.rs b/server/src/kvengine/flushdb.rs index b3e7a97b..5691a6bd 100644 --- a/server/src/kvengine/flushdb.rs +++ b/server/src/kvengine/flushdb.rs @@ -20,7 +20,8 @@ */ use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; /// Delete all the keys in the database diff --git a/server/src/kvengine/get.rs b/server/src/kvengine/get.rs index 86b8c4c4..8a25af6c 100644 --- a/server/src/kvengine/get.rs +++ b/server/src/kvengine/get.rs @@ -23,7 +23,8 @@ //! This module provides functions to work with `GET` queries use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::{BytesWrapper, GroupBegin}; use bytes::Bytes; use libtdb::TResult; diff --git a/server/src/kvengine/jget.rs b/server/src/kvengine/jget.rs index 99cbe486..0309698b 100644 --- a/server/src/kvengine/jget.rs +++ b/server/src/kvengine/jget.rs @@ -24,7 +24,8 @@ //! Functions for handling `JGET` queries use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; /// Run a `JGET` query diff --git a/server/src/kvengine/keylen.rs b/server/src/kvengine/keylen.rs index d4caf0fa..d424f798 100644 --- a/server/src/kvengine/keylen.rs +++ b/server/src/kvengine/keylen.rs @@ -19,7 +19,8 @@ * */ use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; diff --git a/server/src/kvengine/mget.rs b/server/src/kvengine/mget.rs index ac357e04..8345c37f 100644 --- a/server/src/kvengine/mget.rs +++ b/server/src/kvengine/mget.rs @@ -20,7 +20,8 @@ */ use crate::coredb::CoreDB; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::{BytesWrapper, GroupBegin}; use bytes::Bytes; use libtdb::terrapipe::RespCodes; diff --git a/server/src/kvengine/mod.rs b/server/src/kvengine/mod.rs index 68762077..46492892 100644 --- a/server/src/kvengine/mod.rs +++ b/server/src/kvengine/mod.rs @@ -41,9 +41,10 @@ pub mod heya { //! Respond to `HEYA` queries use crate::protocol; use crate::protocol::ActionGroup; + use crate::protocol::Connection; use crate::CoreDB; use libtdb::TResult; - use protocol::{responses, Connection}; + use protocol::responses; /// Returns a `HEY!` `Response` pub async fn heya(_db: &CoreDB, con: &mut Connection, _buf: ActionGroup) -> TResult<()> { con.write_response(&**responses::fresp::R_HEYA).await diff --git a/server/src/kvengine/mset.rs b/server/src/kvengine/mset.rs index fb0e692d..69dba245 100644 --- a/server/src/kvengine/mset.rs +++ b/server/src/kvengine/mset.rs @@ -20,7 +20,8 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; use std::collections::hash_map::Entry; diff --git a/server/src/kvengine/mupdate.rs b/server/src/kvengine/mupdate.rs index b09bf36e..5bda502b 100644 --- a/server/src/kvengine/mupdate.rs +++ b/server/src/kvengine/mupdate.rs @@ -20,7 +20,8 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; use std::collections::hash_map::Entry; diff --git a/server/src/kvengine/set.rs b/server/src/kvengine/set.rs index 42f5e0e0..c323e478 100644 --- a/server/src/kvengine/set.rs +++ b/server/src/kvengine/set.rs @@ -23,7 +23,8 @@ //! This module provides functions to work with `SET` queries use crate::coredb::{self, CoreDB}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use coredb::Data; use libtdb::TResult; use std::collections::hash_map::Entry; diff --git a/server/src/kvengine/strong.rs b/server/src/kvengine/strong.rs index e189f133..785ecc63 100644 --- a/server/src/kvengine/strong.rs +++ b/server/src/kvengine/strong.rs @@ -31,7 +31,8 @@ //! Do note that this isn't the same as the gurantees provided by ACID transactions use crate::coredb::{CoreDB, Data}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; use std::hint::unreachable_unchecked; diff --git a/server/src/kvengine/update.rs b/server/src/kvengine/update.rs index 4619eca4..119a3dbe 100644 --- a/server/src/kvengine/update.rs +++ b/server/src/kvengine/update.rs @@ -23,7 +23,8 @@ //! This module provides functions to work with `UPDATE` queries //! use crate::coredb::{self, CoreDB}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use coredb::Data; use libtdb::TResult; use std::collections::hash_map::Entry; diff --git a/server/src/kvengine/uset.rs b/server/src/kvengine/uset.rs index cf02d806..5d6022a6 100644 --- a/server/src/kvengine/uset.rs +++ b/server/src/kvengine/uset.rs @@ -20,7 +20,8 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::Connection; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; diff --git a/server/src/main.rs b/server/src/main.rs index 75334b46..9389552a 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -38,7 +38,6 @@ mod kvengine; mod protocol; mod queryengine; mod resp; -mod tls; use coredb::CoreDB; use dbnet::run; use env_logger::*; diff --git a/server/src/protocol/mod.rs b/server/src/protocol/mod.rs index abfba009..b740c7c8 100644 --- a/server/src/protocol/mod.rs +++ b/server/src/protocol/mod.rs @@ -38,6 +38,7 @@ use std::io::Result as IoResult; use std::net::SocketAddr; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter}; use tokio::net::TcpStream; +pub mod tls; /// A TCP connection wrapper pub struct Connection { diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs new file mode 100644 index 00000000..b0a0d618 --- /dev/null +++ b/server/src/protocol/tls.rs @@ -0,0 +1,274 @@ +/* + * Created on Fri Dec 18 2020 + * + * This file is a part of TerrabaseDB + * Copyright (c) 2020, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use super::deserializer; +use super::responses; +use super::IoResult; +use super::ParseResult; +use super::QueryResult; +#[allow(dead_code)] // TODO: Don't keep clippy quiet! +use crate::dbnet::Terminator; +use crate::resp::Writable; +use crate::CoreDB; +use bytes::Buf; +use bytes::BytesMut; +use libtdb::TResult; +use libtdb::BUF_CAP; +use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod}; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::io::BufWriter; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::sync::Semaphore; +use tokio::sync::{broadcast, mpsc}; +use tokio::time::{self, Duration}; +use tokio_openssl::SslStream; + +pub struct SslListener { + /// An atomic reference to the coretable + db: CoreDB, + /// The incoming connection listener (binding) + listener: TcpListener, + /// The maximum number of connections + climit: Arc, + /// The shutdown broadcaster + signal: broadcast::Sender<()>, + // When all `Sender`s are dropped - the `Receiver` gets a `None` value + // We send a clone of `terminate_tx` to each `CHandler` + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + acceptor: Arc, +} + +impl SslListener { + pub fn new_pem_based_ssl_connection( + key_file: String, + chain_file: String, + db: CoreDB, + listener: TcpListener, + climit: Arc, + signal: broadcast::Sender<()>, + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + ) -> TResult { + let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?; + acceptor.set_private_key_file(key_file, SslFiletype::PEM)?; + acceptor.set_certificate_chain_file(chain_file)?; + let acceptor = Arc::new(acceptor.build()); + Ok(SslListener { + db, + listener, + climit, + signal, + terminate_tx, + terminate_rx, + acceptor, + }) + } + async fn accept(&mut self) -> TResult> { + let mut backoff = 1; + loop { + match self.listener.accept().await { + // We don't need the bindaddr + // We get the encrypted stream which we need to decrypt + // by using the acceptor + Ok((encrypted_stream, _)) => { + let decrypted_stream = + tokio_openssl::accept(&self.acceptor, encrypted_stream).await?; + return Ok(decrypted_stream); + } + Err(e) => { + if backoff > 64 { + // Too many retries, goodbye user + return Err(e.into()); + } + } + } + // Wait for the `backoff` duration + time::sleep(Duration::from_secs(backoff)).await; + // We're using exponential backoff + backoff *= 2; + } + } + pub async fn run(&mut self) -> TResult<()> { + loop { + // Take the permit first, but we won't use it right now + // that's why we will forget it + self.climit.acquire().await.forget(); + let stream = self.accept().await?; + let mut sslhandle = SslConnectionHandler { + db: self.db.clone(), + con: SslConnection::new(stream), + climit: self.climit.clone(), + terminator: Terminator::new(self.signal.subscribe()), + _term_sig_tx: self.terminate_tx.clone(), + }; + tokio::spawn(async move { + if let Err(e) = sslhandle.run().await { + eprintln!("Error: {}", e); + } + }); + } + } +} + +pub struct SslConnectionHandler { + db: CoreDB, + con: SslConnection, + climit: Arc, + terminator: Terminator, + _term_sig_tx: mpsc::Sender<()>, +} + +impl SslConnectionHandler { + pub async fn run(&mut self) -> TResult<()> { + while !self.terminator.is_termination_signal() { + let try_df = tokio::select! { + tdf = self.con.read_query() => tdf, + _ = self.terminator.receive_signal() => { + return Ok(()); + } + }; + match try_df { + Ok(QueryResult::Q(s)) => self.db.execute_query_ssl(s, &mut self.con).await?, + Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, + Ok(QueryResult::Empty) => return Ok(()), + Err(e) => return Err(e.into()), + } + } + Ok(()) + } +} + +pub struct SslConnection { + stream: BufWriter>, + buffer: BytesMut, +} + +impl SslConnection { + pub fn new(stream: SslStream) -> Self { + SslConnection { + stream: BufWriter::new(stream), + buffer: BytesMut::with_capacity(BUF_CAP), + } + } + async fn read_again(&mut self) -> Result<(), String> { + match self.stream.read_buf(&mut self.buffer).await { + Ok(0) => { + // If 0 bytes were received, then the remote end closed + // the connection + if self.buffer.is_empty() { + return Ok(()); + } else { + return Err(format!( + "Connection reset while reading from {}", + if let Ok(p) = self.get_peer() { + p.to_string() + } else { + "peer".to_owned() + } + ) + .into()); + } + } + Ok(_) => Ok(()), + Err(e) => return Err(format!("{}", e)), + } + } + fn get_peer(&self) -> IoResult { + self.stream.get_ref().get_ref().peer_addr() + } + /// Try to parse a query from the buffered data + fn try_query(&mut self) -> Result { + if self.buffer.is_empty() { + return Err(()); + } + Ok(deserializer::parse(&self.buffer)) + } + pub async fn read_query(&mut self) -> Result { + self.read_again().await?; + loop { + match self.try_query() { + Ok(ParseResult::Query(query, forward)) => { + self.buffer.advance(forward); + return Ok(QueryResult::Q(query)); + } + Ok(ParseResult::BadPacket(bp)) => { + self.buffer.advance(bp); + return Ok(QueryResult::E(responses::fresp::R_PACKET_ERR.to_owned())); + } + Err(_) => { + return Ok(QueryResult::Empty); + } + _ => (), + } + self.read_again().await?; + } + } + /// Write a response to the stream + pub async fn write_response(&mut self, streamer: impl Writable) -> TResult<()> { + streamer.write(&mut self.stream).await?; + Ok(()) + } + pub async fn flush_stream(&mut self) -> TResult<()> { + self.stream.flush().await?; + Ok(()) + } + /// Wraps around the `write_response` used to differentiate between a + /// success response and an error response + pub async fn close_conn_with_error(&mut self, resp: Vec) -> TResult<()> { + self.write_response(resp).await?; + self.stream.flush().await?; + Ok(()) + } +} + +/// A per-connection handler +pub struct SslCHandler { + db: CoreDB, + con: SslConnection, + climit: Arc, + terminator: Terminator, + _term_sig_tx: mpsc::Sender<()>, +} + +impl SslCHandler { + /// Process the incoming connection + pub async fn run(&mut self) -> TResult<()> { + while !self.terminator.is_termination_signal() { + let try_df = tokio::select! { + tdf = self.con.read_query() => tdf, + _ = self.terminator.receive_signal() => { + return Ok(()); + } + }; + match try_df { + Ok(QueryResult::Q(s)) => self.db.execute_query_ssl(s, &mut self.con).await?, + Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, + Ok(QueryResult::Empty) => return Ok(()), + Err(e) => return Err(e.into()), + } + } + Ok(()) + } +} diff --git a/server/src/resp/mod.rs b/server/src/resp/mod.rs index 799704c1..ff6b043b 100644 --- a/server/src/resp/mod.rs +++ b/server/src/resp/mod.rs @@ -21,15 +21,18 @@ //! Utilities for generating responses, which are only used by the `server` //! - +use crate::protocol::tls::SslConnection; +use crate::protocol::Connection; use bytes::Bytes; use libtdb::terrapipe::RespCodes; use std::error::Error; use std::future::Future; +use std::io::Error as IoError; use std::pin::Pin; use tokio::io::AsyncWriteExt; use tokio::io::BufWriter; use tokio::net::TcpStream; +use tokio_openssl::SslStream; /// # The `Writable` trait /// All trait implementors are given access to an asynchronous stream to which @@ -42,10 +45,35 @@ pub trait Writable { */ fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's>>; } +pub trait IsConnection: std::marker::Sync + std::marker::Send { + fn write_lowlevel<'s>( + &'s mut self, + bytes: &'s [u8], + ) -> Pin> + Send + Sync + 's>>; +} + +impl IsConnection for BufWriter { + fn write_lowlevel<'s>( + &'s mut self, + bytes: &'s [u8], + ) -> Pin> + Send + Sync + 's>> { + Box::pin(self.write(bytes)) + } +} + +impl IsConnection for BufWriter> { + fn write_lowlevel<'s>( + &'s mut self, + bytes: &'s [u8], + ) -> Pin> + Send + Sync + 's>> { + Box::pin(self.write(bytes)) + } +} + /// A `BytesWrapper` object wraps around a `Bytes` object that might have been pulled /// from `CoreDB`. /// @@ -73,14 +101,14 @@ impl BytesWrapper { impl Writable for Vec { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, resp: Vec, ) -> Result<(), Box> { - con.write(&resp).await?; + con.write_lowlevel(&resp).await?; Ok(()) } Box::pin(write_bytes(con, self)) @@ -90,14 +118,14 @@ impl Writable for Vec { impl Writable for &'static [u8] { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, resp: &[u8], ) -> Result<(), Box> { - con.write(&resp).await?; + con.write_lowlevel(&resp).await?; Ok(()) } Box::pin(write_bytes(con, &self)) @@ -107,28 +135,28 @@ impl Writable for &'static [u8] { impl Writable for BytesWrapper { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, bytes: Bytes, ) -> Result<(), Box> { // First write a `+` character to the stream since this is a // string (we represent `String`s as `Byte` objects internally) // and since `Bytes` are effectively `String`s we will append the // type operator `+` to the stream - con.write(&[b'+']).await?; + con.write_lowlevel(&[b'+']).await?; // Now get the size of the Bytes object as bytes let size = bytes.len().to_string().into_bytes(); // Write this to the stream - con.write(&size).await?; + con.write_lowlevel(&size).await?; // Now write a LF character - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; // Now write the REAL bytes (of the object) - con.write(&bytes).await?; + con.write_lowlevel(&bytes).await?; // Now write another LF - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; Ok(()) } Box::pin(write_bytes(con, self.finish_into_bytes())) @@ -138,44 +166,44 @@ impl Writable for BytesWrapper { impl Writable for RespCodes { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, code: RespCodes, ) -> Result<(), Box> { if let RespCodes::OtherError(Some(e)) = code { // Since this is an other error which contains a description // we'll write ! followed by the string - con.write(&[b'!']).await?; + con.write_lowlevel(&[b'!']).await?; // Convert the string into a vector of bytes let e = e.to_string().into_bytes(); // Now get the length of the byte vector and turn it into // a string and then into a byte vector let len_as_bytes = e.len().to_string().into_bytes(); // Write the length - con.write(&len_as_bytes).await?; + con.write_lowlevel(&len_as_bytes).await?; // Then an LF - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; // Then the error string - con.write(&e).await?; + con.write_lowlevel(&e).await?; // Then another LF - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; // And now we're done return Ok(()); } // Self's tsymbol is ! // The length of the response code is 1 // And we need a newline - con.write(&[b'!', b'1', b'\n']).await?; + con.write_lowlevel(&[b'!', b'1', b'\n']).await?; // We need to get the u8 version of the response code let code: u8 = code.into(); // We need the UTF8 equivalent of the response code let code_bytes = code.to_string().into_bytes(); - con.write(&code_bytes).await?; + con.write_lowlevel(&code_bytes).await?; // Now append a newline - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; Ok(()) } Box::pin(write_bytes(con, self)) @@ -185,27 +213,27 @@ impl Writable for RespCodes { impl Writable for GroupBegin { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, size: usize, ) -> Result<(), Box> { - con.write(b"#2\n*1\n").await?; + con.write_lowlevel(b"#2\n*1\n").await?; // First write a `#` which indicates that the next bytes give the // prefix length - con.write(&[b'#']).await?; + con.write_lowlevel(&[b'#']).await?; let group_len_as_bytes = size.to_string().into_bytes(); let group_prefix_len_as_bytes = (group_len_as_bytes.len() + 1).to_string().into_bytes(); // Now write Self's len as bytes - con.write(&group_prefix_len_as_bytes).await?; + con.write_lowlevel(&group_prefix_len_as_bytes).await?; // Now write a LF and '&' which signifies the beginning of a datagroup - con.write(&[b'\n', b'&']).await?; + con.write_lowlevel(&[b'\n', b'&']).await?; // Now write the number of items in the datagroup as bytes - con.write(&group_len_as_bytes).await?; + con.write_lowlevel(&group_len_as_bytes).await?; // Now write a '\n' character - con.write(&[b'\n']).await?; + con.write_lowlevel(&[b'\n']).await?; Ok(()) } Box::pin(write_bytes(con, self.0)) @@ -215,20 +243,20 @@ impl Writable for GroupBegin { impl Writable for usize { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { async fn write_bytes( - con: &mut BufWriter, + con: &mut impl IsConnection, val: usize, ) -> Result<(), Box> { - con.write(b":").await?; + con.write_lowlevel(b":").await?; let usize_bytes = val.to_string().into_bytes(); let usize_bytes_len = usize_bytes.len().to_string().into_bytes(); - con.write(&usize_bytes_len).await?; - con.write(b"\n").await?; - con.write(&usize_bytes).await?; - con.write(b"\n").await?; + con.write_lowlevel(&usize_bytes_len).await?; + con.write_lowlevel(b"\n").await?; + con.write_lowlevel(&usize_bytes).await?; + con.write_lowlevel(b"\n").await?; Ok(()) } Box::pin(write_bytes(con, self)) @@ -238,20 +266,17 @@ impl Writable for usize { impl Writable for u64 { fn write<'s>( self, - con: &'s mut BufWriter, + con: &'s mut impl IsConnection, ) -> Pin>> + Send + Sync + 's)>> { - async fn write_bytes( - con: &mut BufWriter, - val: u64, - ) -> Result<(), Box> { - con.write(b":").await?; + async fn write_bytes(con: &mut impl IsConnection, val: u64) -> Result<(), Box> { + con.write_lowlevel(b":").await?; let usize_bytes = val.to_string().into_bytes(); let usize_bytes_len = usize_bytes.len().to_string().into_bytes(); - con.write(&usize_bytes_len).await?; - con.write(b"\n").await?; - con.write(&usize_bytes).await?; - con.write(b"\n").await?; + con.write_lowlevel(&usize_bytes_len).await?; + con.write_lowlevel(b"\n").await?; + con.write_lowlevel(&usize_bytes).await?; + con.write_lowlevel(b"\n").await?; Ok(()) } Box::pin(write_bytes(con, self)) diff --git a/server/src/tls/mod.rs b/server/src/tls/mod.rs deleted file mode 100644 index 02e73c72..00000000 --- a/server/src/tls/mod.rs +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Created on Fri Dec 18 2020 - * - * This file is a part of TerrabaseDB - * Copyright (c) 2020, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -#[allow(dead_code)] // TODO: Don't keep clippy quiet! -// use crate::dbnet::CHandler; -// use crate::dbnet::Terminator; -// use crate::protocol::Connection; -use crate::CoreDB; -use libtdb::TResult; -use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod}; -// use std::net::ToSocketAddrs; -use std::sync::Arc; -// use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::net::TcpListener; -use tokio::net::TcpStream; -use tokio::sync::Semaphore; -use tokio::sync::{broadcast, mpsc}; -use tokio::time::{self, Duration}; -use tokio_openssl_vendored::SslStream; -pub struct SslListener { - /// An atomic reference to the coretable - db: CoreDB, - /// The incoming connection listener (binding) - listener: TcpListener, - /// The maximum number of connections - climit: Arc, - /// The shutdown broadcaster - signal: broadcast::Sender<()>, - // When all `Sender`s are dropped - the `Receiver` gets a `None` value - // We send a clone of `terminate_tx` to each `CHandler` - terminate_tx: mpsc::Sender<()>, - terminate_rx: mpsc::Receiver<()>, - acceptor: Arc, -} - -impl SslListener { - pub fn new_pem_based_ssl_connection( - key_file: String, - chain_file: String, - db: CoreDB, - listener: TcpListener, - climit: Arc, - signal: broadcast::Sender<()>, - terminate_tx: mpsc::Sender<()>, - terminate_rx: mpsc::Receiver<()>, - ) -> TResult { - let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?; - acceptor.set_private_key_file(key_file, SslFiletype::PEM)?; - acceptor.set_certificate_chain_file(chain_file)?; - let acceptor = Arc::new(acceptor.build()); - Ok(SslListener { - db, - listener, - climit, - signal, - terminate_tx, - terminate_rx, - acceptor, - }) - } - async fn accept(&mut self) -> TResult> { - let mut backoff = 1; - loop { - match self.listener.accept().await { - // We don't need the bindaddr - // We get the encrypted stream which we need to decrypt - // by using the acceptor - Ok((encrypted_stream, _)) => { - let decrypted_stream = - tokio_openssl_vendored::accept(&self.acceptor, encrypted_stream).await?; - return Ok(decrypted_stream); - } - Err(e) => { - if backoff > 64 { - // Too many retries, goodbye user - return Err(e.into()); - } - } - } - // Wait for the `backoff` duration - time::sleep(Duration::from_secs(backoff)).await; - // We're using exponential backoff - backoff *= 2; - } - } -} From 25d211d0a282d6400570624531925ff6f94f5786 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sun, 27 Dec 2020 20:31:27 +0530 Subject: [PATCH 03/17] Implement `Con<'_>` object for TLS/non-TLS streams This object either holds a mutable reference to an unencrypted TCP stream or an encrypted TLS stream (using OpenSSL). The action handlers were modified as a consequence. Signed-off-by: Sayan Nandan --- server/src/admin/mksnap.rs | 5 ++-- server/src/coredb/mod.rs | 3 +- server/src/dbnet.rs | 51 +++++++++++++++++++++++++++++++++- server/src/kvengine/dbsize.rs | 4 +-- server/src/kvengine/del.rs | 4 +-- server/src/kvengine/exists.rs | 4 +-- server/src/kvengine/flushdb.rs | 4 +-- server/src/kvengine/get.rs | 4 +-- server/src/kvengine/jget.rs | 4 +-- server/src/kvengine/keylen.rs | 4 +-- server/src/kvengine/mget.rs | 4 +-- server/src/kvengine/mod.rs | 4 +-- server/src/kvengine/mset.rs | 4 +-- server/src/kvengine/mupdate.rs | 4 +-- server/src/kvengine/set.rs | 4 +-- server/src/kvengine/strong.rs | 8 +++--- server/src/kvengine/update.rs | 4 +-- server/src/kvengine/uset.rs | 4 +-- server/src/queryengine/mod.rs | 3 +- 19 files changed, 89 insertions(+), 37 deletions(-) diff --git a/server/src/admin/mksnap.rs b/server/src/admin/mksnap.rs index 07a8b55d..f5dce926 100644 --- a/server/src/admin/mksnap.rs +++ b/server/src/admin/mksnap.rs @@ -20,10 +20,11 @@ */ use crate::coredb::CoreDB; +use crate::dbnet::Con; use crate::diskstore; use crate::diskstore::snapshot::SnapshotEngine; use crate::diskstore::snapshot::DIR_SNAPSHOT; -use crate::protocol::{responses, ActionGroup, Connection}; +use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::terrapipe::RespCodes; use libtdb::TResult; @@ -32,7 +33,7 @@ use std::path::PathBuf; /// Create a snapshot /// -pub async fn mksnap(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn mksnap(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany == 0 { if !handle.is_snapshot_enabled() { diff --git a/server/src/coredb/mod.rs b/server/src/coredb/mod.rs index 14051aa2..2f5e5645 100644 --- a/server/src/coredb/mod.rs +++ b/server/src/coredb/mod.rs @@ -21,6 +21,7 @@ //! # The core database engine +use crate::dbnet::Con; use crate::config::BGSave; use crate::config::SnapshotConfig; use crate::config::SnapshotPref; @@ -245,7 +246,7 @@ impl CoreDB { } /// Execute a query that has already been validated by `Connection::read_query` - pub async fn execute_query(&self, query: Query, mut con: &mut Connection) -> TResult<()> { + pub async fn execute_query(&self, query: Query, mut con: &mut Con<'_>) -> TResult<()> { match query { Query::Simple(q) => { queryengine::execute_simple(&self, &mut con, q).await?; diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index aaf70309..2c344afd 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -37,7 +37,9 @@ use crate::config::BGSave; use crate::config::SnapshotConfig; use crate::diskstore::snapshot::DIR_REMOTE_SNAPSHOT; +use crate::protocol::tls::SslConnection; use crate::protocol::{Connection, QueryResult::*}; +use crate::resp::Writable; use crate::CoreDB; use libtdb::util::terminal; use libtdb::TResult; @@ -156,6 +158,49 @@ impl Listener { } } +/// # Connection Wrapper +/// +/// A `Con` object holds a mutable reference to a standard TCP stream or to +/// an encrypted connection (over the `SslListener` object). It provides a few +/// methods which are provided by the underlying interface. +pub enum Con<'a> { + /// A secure TLS connection + Secure(&'a mut SslConnection), + /// An insecure ('standard') TCP connection + Insecure(&'a mut Connection), +} + +impl<'a> Con<'a> { + /// Create a new **unencrypted** connection instance + pub fn init<'b>(con: &'b mut Connection) -> Self + where + 'b: 'a, + { + Con::Insecure(con) + } + /// Create a new **encrypted** connection instance + pub fn init_ssl<'b>(con: &'b mut SslConnection) -> Self + where + 'b: 'a, + { + Con::Secure(con) + } + /// Flush the stream that is held by the underlying connection + pub async fn flush_stream(&mut self) -> TResult<()> { + match self { + Con::Secure(con) => con.flush_stream().await, + Con::Insecure(con) => con.flush_stream().await, + } + } + /// Write bytes to the underlying stream that implement the `Writable` trait + pub async fn write_response(&mut self, resp: impl Writable) -> TResult<()> { + match self { + Con::Insecure(con) => con.write_response(resp).await, + Con::Secure(con) => con.write_response(resp).await, + } + } +} + impl CHandler { /// Process the incoming connection pub async fn run(&mut self) -> TResult<()> { @@ -167,7 +212,11 @@ impl CHandler { } }; match try_df { - Ok(Q(s)) => self.db.execute_query(s, &mut self.con).await?, + Ok(Q(s)) => { + self.db + .execute_query(s, &mut Con::Insecure(&mut self.con)) + .await? + } Ok(E(r)) => self.con.close_conn_with_error(r).await?, Ok(Empty) => return Ok(()), Err(e) => return Err(e.into()), diff --git a/server/src/kvengine/dbsize.rs b/server/src/kvengine/dbsize.rs index 192392f6..a77f69f8 100644 --- a/server/src/kvengine/dbsize.rs +++ b/server/src/kvengine/dbsize.rs @@ -19,13 +19,13 @@ * */ use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; /// Get the number of keys in the database -pub async fn dbsize(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn dbsize(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { if act.howmany() != 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; } diff --git a/server/src/kvengine/del.rs b/server/src/kvengine/del.rs index 6437ac8c..ead67647 100644 --- a/server/src/kvengine/del.rs +++ b/server/src/kvengine/del.rs @@ -23,7 +23,7 @@ //! This module provides functions to work with `DEL` queries use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; @@ -32,7 +32,7 @@ use libtdb::TResult; /// /// Do note that this function is blocking since it acquires a write lock. /// It will write an entire datagroup, for this `del` action -pub async fn del(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn del(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/exists.rs b/server/src/kvengine/exists.rs index 9ffe055a..c176d6b2 100644 --- a/server/src/kvengine/exists.rs +++ b/server/src/kvengine/exists.rs @@ -23,13 +23,13 @@ //! This module provides functions to work with `EXISTS` queries use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; /// Run an `EXISTS` query -pub async fn exists(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn exists(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/flushdb.rs b/server/src/kvengine/flushdb.rs index 5691a6bd..5a3fbdd8 100644 --- a/server/src/kvengine/flushdb.rs +++ b/server/src/kvengine/flushdb.rs @@ -20,12 +20,12 @@ */ use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; /// Delete all the keys in the database -pub async fn flushdb(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn flushdb(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { if act.howmany() != 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; } diff --git a/server/src/kvengine/get.rs b/server/src/kvengine/get.rs index 8a25af6c..723f6d41 100644 --- a/server/src/kvengine/get.rs +++ b/server/src/kvengine/get.rs @@ -23,14 +23,14 @@ //! This module provides functions to work with `GET` queries use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::{BytesWrapper, GroupBegin}; use bytes::Bytes; use libtdb::TResult; /// Run a `GET` query -pub async fn get(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn get(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany != 1 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/jget.rs b/server/src/kvengine/jget.rs index 0309698b..709bcb7b 100644 --- a/server/src/kvengine/jget.rs +++ b/server/src/kvengine/jget.rs @@ -24,7 +24,7 @@ //! Functions for handling `JGET` queries use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; @@ -37,7 +37,7 @@ use libtdb::TResult; /// {"key":"value"}\n /// ``` /// -pub async fn jget(_handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn jget(_handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany != 1 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/keylen.rs b/server/src/kvengine/keylen.rs index d424f798..bc9c816c 100644 --- a/server/src/kvengine/keylen.rs +++ b/server/src/kvengine/keylen.rs @@ -19,7 +19,7 @@ * */ use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; @@ -27,7 +27,7 @@ use libtdb::TResult; /// Run a `KEYLEN` query /// /// At this moment, `keylen` only supports a single key -pub async fn keylen(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn keylen(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany != 1 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/mget.rs b/server/src/kvengine/mget.rs index 8345c37f..65fda7f6 100644 --- a/server/src/kvengine/mget.rs +++ b/server/src/kvengine/mget.rs @@ -20,7 +20,7 @@ */ use crate::coredb::CoreDB; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::{BytesWrapper, GroupBegin}; use bytes::Bytes; @@ -29,7 +29,7 @@ use libtdb::TResult; /// Run an `MGET` query /// -pub async fn mget(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn mget(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/mod.rs b/server/src/kvengine/mod.rs index 46492892..a1f94db0 100644 --- a/server/src/kvengine/mod.rs +++ b/server/src/kvengine/mod.rs @@ -39,14 +39,14 @@ pub mod update; pub mod uset; pub mod heya { //! Respond to `HEYA` queries + use crate::dbnet::Con; use crate::protocol; use crate::protocol::ActionGroup; - use crate::protocol::Connection; use crate::CoreDB; use libtdb::TResult; use protocol::responses; /// Returns a `HEY!` `Response` - pub async fn heya(_db: &CoreDB, con: &mut Connection, _buf: ActionGroup) -> TResult<()> { + pub async fn heya(_db: &CoreDB, con: &mut Con<'_>, _buf: ActionGroup) -> TResult<()> { con.write_response(&**responses::fresp::R_HEYA).await } } diff --git a/server/src/kvengine/mset.rs b/server/src/kvengine/mset.rs index 69dba245..7496ccf2 100644 --- a/server/src/kvengine/mset.rs +++ b/server/src/kvengine/mset.rs @@ -20,14 +20,14 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; use std::collections::hash_map::Entry; /// Run an `MSET` query -pub async fn mset(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn mset(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany & 1 == 1 || howmany == 0 { // An odd number of arguments means that the number of keys diff --git a/server/src/kvengine/mupdate.rs b/server/src/kvengine/mupdate.rs index 5bda502b..04d0c42e 100644 --- a/server/src/kvengine/mupdate.rs +++ b/server/src/kvengine/mupdate.rs @@ -20,14 +20,14 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; use std::collections::hash_map::Entry; /// Run an `MUPDATE` query -pub async fn mupdate(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn mupdate(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany & 1 == 1 || howmany == 0 { // An odd number of arguments means that the number of keys diff --git a/server/src/kvengine/set.rs b/server/src/kvengine/set.rs index c323e478..a0ad73bb 100644 --- a/server/src/kvengine/set.rs +++ b/server/src/kvengine/set.rs @@ -23,7 +23,7 @@ //! This module provides functions to work with `SET` queries use crate::coredb::{self, CoreDB}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use coredb::Data; use libtdb::TResult; @@ -31,7 +31,7 @@ use std::collections::hash_map::Entry; use std::hint::unreachable_unchecked; /// Run a `SET` query -pub async fn set(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn set(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany != 2 { // There should be exactly 2 arguments diff --git a/server/src/kvengine/strong.rs b/server/src/kvengine/strong.rs index 785ecc63..99bbfe61 100644 --- a/server/src/kvengine/strong.rs +++ b/server/src/kvengine/strong.rs @@ -31,7 +31,7 @@ //! Do note that this isn't the same as the gurantees provided by ACID transactions use crate::coredb::{CoreDB, Data}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use libtdb::TResult; use std::hint::unreachable_unchecked; @@ -40,7 +40,7 @@ use std::hint::unreachable_unchecked; /// /// This either returns `Okay` if all the keys were set, or it returns an /// `Overwrite Error` or code `2` -pub async fn sset(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn sset(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany & 1 == 1 || howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; @@ -102,7 +102,7 @@ pub async fn sset(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TR /// /// This either returns `Okay` if all the keys were `del`eted, or it returns a /// `Nil`, which is code `1` -pub async fn sdel(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn sdel(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; @@ -158,7 +158,7 @@ pub async fn sdel(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TR /// /// This either returns `Okay` if all the keys were updated, or it returns `Nil` /// or code `1` -pub async fn supdate(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn supdate(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany & 1 == 1 || howmany == 0 { return con.write_response(&**responses::fresp::R_ACTION_ERR).await; diff --git a/server/src/kvengine/update.rs b/server/src/kvengine/update.rs index 119a3dbe..1eeed75b 100644 --- a/server/src/kvengine/update.rs +++ b/server/src/kvengine/update.rs @@ -23,7 +23,7 @@ //! This module provides functions to work with `UPDATE` queries //! use crate::coredb::{self, CoreDB}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use coredb::Data; use libtdb::TResult; @@ -31,7 +31,7 @@ use std::collections::hash_map::Entry; use std::hint::unreachable_unchecked; /// Run an `UPDATE` query -pub async fn update(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn update(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany != 2 { // There should be exactly 2 arguments diff --git a/server/src/kvengine/uset.rs b/server/src/kvengine/uset.rs index 5d6022a6..37235a1a 100644 --- a/server/src/kvengine/uset.rs +++ b/server/src/kvengine/uset.rs @@ -20,7 +20,7 @@ */ use crate::coredb::{self, CoreDB}; -use crate::protocol::Connection; +use crate::dbnet::Con; use crate::protocol::{responses, ActionGroup}; use crate::resp::GroupBegin; use libtdb::TResult; @@ -28,7 +28,7 @@ use libtdb::TResult; /// Run an `USET` query /// /// This is like "INSERT or UPDATE" -pub async fn uset(handle: &CoreDB, con: &mut Connection, act: ActionGroup) -> TResult<()> { +pub async fn uset(handle: &CoreDB, con: &mut Con<'_>, act: ActionGroup) -> TResult<()> { let howmany = act.howmany(); if howmany & 1 == 1 || howmany == 0 { // An odd number of arguments means that the number of keys diff --git a/server/src/queryengine/mod.rs b/server/src/queryengine/mod.rs index c732ed92..df5903df 100644 --- a/server/src/queryengine/mod.rs +++ b/server/src/queryengine/mod.rs @@ -22,6 +22,7 @@ //! # The Query Engine use crate::coredb::CoreDB; +use crate::dbnet::Con; use crate::protocol::ActionGroup; use crate::protocol::{responses, Connection}; use crate::{admin, kvengine}; @@ -66,7 +67,7 @@ mod tags { } /// Execute a simple(*) query -pub async fn execute_simple(db: &CoreDB, con: &mut Connection, buf: ActionGroup) -> TResult<()> { +pub async fn execute_simple(db: &CoreDB, con: &mut Con<'_>, buf: ActionGroup) -> TResult<()> { let first = match buf.get_first() { None => { return con From 61daf6c2e29471ef4be436865597980be9141a02 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Tue, 5 Jan 2021 13:46:44 +0530 Subject: [PATCH 04/17] Add example on TLS configuration Also, we've muted dead_code warning froms the compiler, temporarily. Signed-off-by: Sayan Nandan --- examples/config-files/ssl.toml | 16 ++++++++++++++++ server/src/protocol/tls.rs | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 examples/config-files/ssl.toml diff --git a/examples/config-files/ssl.toml b/examples/config-files/ssl.toml new file mode 100644 index 00000000..9c74046a --- /dev/null +++ b/examples/config-files/ssl.toml @@ -0,0 +1,16 @@ +[server] +host = "127.0.0.1" +port = 2003 +noart = false + +[ssl] +key = "/path/to/keyfile.pem" +chain = "/path/to/chain.pem" + +[bgsave] +enabled = true +every = 120 + +[snapshot] +every = 3600 +atmost = 4 \ No newline at end of file diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index b0a0d618..2fc9da87 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -19,12 +19,12 @@ * */ +#![allow(dead_code)] // TODO: Don't keep clippy quiet! use super::deserializer; use super::responses; use super::IoResult; use super::ParseResult; use super::QueryResult; -#[allow(dead_code)] // TODO: Don't keep clippy quiet! use crate::dbnet::Terminator; use crate::resp::Writable; use crate::CoreDB; From 442570a58b7faa7e10e64004ac55621e6fe70b2a Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Thu, 7 Jan 2021 13:52:04 +0530 Subject: [PATCH 05/17] Let SSL opts be read from the cfg file and cli This commit enables SSL settings to be read from command-line arguments and from the configuration file. Signed-off-by: Sayan Nandan --- server/src/cli.yml | 19 +++++++ server/src/config/mod.rs | 114 +++++++++++++++++++++++++++++++++---- server/src/protocol/tls.rs | 1 - 3 files changed, 122 insertions(+), 12 deletions(-) diff --git a/server/src/cli.yml b/server/src/cli.yml index 04ba4cdf..c4834a55 100644 --- a/server/src/cli.yml +++ b/server/src/cli.yml @@ -81,3 +81,22 @@ args: value_name: count help: Sets the number of most recent snapshots to keep takes_value: true + - sslkey: + required: false + long: sslkey + short: k + value_name: key + help: Sets the PEM key file to use for SSL/TLS + takes_value: true + - sslchain: + required: false + long: sslchain + short: z + value_name: chain + help: Sets the PEM chain file to use for SSL/TLS + takes_value: true + - sslonly: + required: false + long: sslonly + takes_value: false + help: Tells the server to only accept SSL connections and disables the non-SSL port diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index aa1a5809..73932653 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -48,6 +48,8 @@ pub struct Config { bgsave: Option, /// The snapshot key snapshot: Option, + /// SSL configuration + ssl: Option, } /// The BGSAVE section in the config file @@ -124,6 +126,41 @@ pub struct ConfigKeySnapshot { atmost: usize, } +/// The SSL configuration +/// +/// This enumeration determines whether the SSL listener is: +/// - `Enabled`: This means that the database server will be listening to both +/// SSL **and** non-SSL requests +/// - `EnabledOnly` : This means that the database server will only accept SSL requests +/// and will not even activate the non-SSL socket +/// - `Disabled` : This indicates that the server would only accept non-SSL connections +/// and will not even activate the SSL socket +#[derive(Debug, PartialEq)] +pub enum SslConfig { + Enabled(SslOpts), + EnabledOnly(SslOpts), + Disabled, +} + +#[derive(Deserialize, Debug, PartialEq)] +pub struct KeySslOpts { + key: String, + chain: String, + only: Option, +} + +#[derive(Deserialize, Debug, PartialEq)] +pub struct SslOpts { + key: String, + chain: String, +} + +impl SslOpts { + pub const fn new(key: String, chain: String) -> Self { + SslOpts { key, chain } + } +} + #[derive(Debug, PartialEq)] /// The snapshot configuration /// @@ -190,6 +227,8 @@ pub struct ParsedConfig { pub bgsave: BGSave, /// The snapshot configuration pub snapshot: SnapshotConfig, + /// The SSL configuration + pub ssl: SslConfig, } impl ParsedConfig { @@ -206,16 +245,16 @@ impl ParsedConfig { } /// Create a `ParsedConfig` instance from a `Config` object, which is a parsed /// TOML file (represented as an object) - const fn from_config(cfg: Config) -> Self { + fn from_config(cfg_info: Config) -> Self { ParsedConfig { - host: cfg.server.host, - port: cfg.server.port, - noart: if let Some(noart) = cfg.server.noart { + host: cfg_info.server.host, + port: cfg_info.server.port, + noart: if let Some(noart) = cfg_info.server.noart { noart } else { false }, - bgsave: if let Some(bgsave) = cfg.bgsave { + bgsave: if let Some(bgsave) = cfg_info.bgsave { match (bgsave.enabled, bgsave.every) { // TODO: Show a warning that there are unused keys (Some(enabled), Some(every)) => BGSave::new(enabled, every), @@ -226,11 +265,26 @@ impl ParsedConfig { } else { BGSave::default() }, - snapshot: if let Some(snapshot) = cfg.snapshot { + snapshot: if let Some(snapshot) = cfg_info.snapshot { SnapshotConfig::Enabled(SnapshotPref::new(snapshot.every, snapshot.atmost)) } else { SnapshotConfig::default() }, + ssl: if let Some(sslopts) = cfg_info.ssl { + if sslopts.only.is_some() { + SslConfig::EnabledOnly(SslOpts { + key: sslopts.key, + chain: sslopts.chain, + }) + } else { + SslConfig::Enabled(SslOpts { + key: sslopts.key, + chain: sslopts.chain, + }) + } + } else { + SslConfig::Disabled + }, } } #[cfg(test)] @@ -247,6 +301,7 @@ impl ParsedConfig { noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled, } } /// Create a new `ParsedConfig` with the default `port` and `noart` settngs @@ -258,6 +313,7 @@ impl ParsedConfig { false, BGSave::default(), SnapshotConfig::default(), + SslConfig::Disabled, ) } /// Create a new `ParsedConfig` with all the fields @@ -267,6 +323,7 @@ impl ParsedConfig { noart: bool, bgsave: BGSave, snapshot: SnapshotConfig, + ssl: SslConfig, ) -> Self { ParsedConfig { host, @@ -274,6 +331,7 @@ impl ParsedConfig { noart, bgsave, snapshot, + ssl, } } /// Create a default `ParsedConfig` with the following setup defaults: @@ -282,6 +340,7 @@ impl ParsedConfig { /// - `noart` : false /// - `bgsave_enabled` : true /// - `bgsave_duration` : 120 + /// - `ssl` : disabled pub const fn default() -> Self { ParsedConfig { host: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), @@ -289,6 +348,7 @@ impl ParsedConfig { noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled, } } /// Return a (host, port) tuple which can be bound to with `TcpListener` @@ -409,13 +469,18 @@ pub fn get_config_file_or_return_cfg() -> Result Result SnapshotConfig::Disabled, }; - let cfg = ParsedConfig::new(host, port, noart, bgsave, snapcfg); + let sslcfg = match (sslkey, sslchain) { + (Some(key), Some(chain)) => { + if sslonly { + SslConfig::EnabledOnly(SslOpts::new(key.to_owned(), chain.to_owned())) + } else { + SslConfig::Enabled(SslOpts::new(key.to_owned(), chain.to_owned())) + } + } + (None, None) => { + if sslonly { + log::warn!("Ignoring --sslonly flag as SSL wasn't enabled") + } + SslConfig::Disabled + } + (_, _) => { + return Err(ConfigError::CliArgErr( + "To use SSL, pass values for both --sslkey and --sslchain", + )); + } + }; + let cfg = ParsedConfig::new(host, port, noart, bgsave, snapcfg, sslcfg); return Ok(ConfigType::Custom(cfg, restorefile)); } if let Some(filename) = filename { @@ -563,6 +648,7 @@ fn test_config_file_noart() { noart: true, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled } ); } @@ -580,6 +666,7 @@ fn test_config_file_ipv6() { noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled } ); } @@ -596,7 +683,8 @@ fn test_config_file_template() { 2003, false, BGSave::default(), - SnapshotConfig::Enabled(SnapshotPref::new(3600, 4)) + SnapshotConfig::Enabled(SnapshotPref::new(3600, 4)), + SslConfig::Disabled // TODO: Update the template ) ); } @@ -621,7 +709,8 @@ fn test_config_file_custom_bgsave() { port: 2003, noart: false, bgsave: BGSave::new(true, 600), - snapshot: SnapshotConfig::default() + snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled } ); } @@ -642,6 +731,7 @@ fn test_config_file_bgsave_enabled_only() { noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled } ) } @@ -661,7 +751,8 @@ fn test_config_file_bgsave_every_only() { port: 2003, noart: false, bgsave: BGSave::new(true, 600), - snapshot: SnapshotConfig::default() + snapshot: SnapshotConfig::default(), + ssl: SslConfig::Disabled } ) } @@ -678,6 +769,7 @@ fn test_config_file_snapshot() { host: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port: 2003, noart: false, + ssl: SslConfig::Disabled } ); } diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 2fc9da87..7d8deca1 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -19,7 +19,6 @@ * */ -#![allow(dead_code)] // TODO: Don't keep clippy quiet! use super::deserializer; use super::responses; use super::IoResult; From b2e3d907993dbcd5a1d0ad6c0dafbf6df923ccbc Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sat, 9 Jan 2021 20:41:58 +0530 Subject: [PATCH 06/17] Fix bug that suppressed unused cli flags warning The cli.yaml file was also reformatted Signed-off-by: Sayan Nandan --- server/src/cli.yml | 175 +++++++++++++++++---------------------- server/src/config/mod.rs | 33 +++++--- 2 files changed, 99 insertions(+), 109 deletions(-) diff --git a/server/src/cli.yml b/server/src/cli.yml index c4834a55..2d7be12f 100644 --- a/server/src/cli.yml +++ b/server/src/cli.yml @@ -1,102 +1,83 @@ -# -# Created on Tue Sep 01 2020 -# -# This file is a part of TerrabaseDB -# Copyright (c) 2020, Sayan Nandan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# - name: TerrabaseDB Server version: 0.5.0 author: Sayan N. about: The TerrabaseDB Database server args: - - config: - short: c - required: false - long: withconfig - value_name: cfgfile - help: Sets a configuration file to start tdb - takes_value: true - - restore: - short: r - required: false - long: restore - value_name: snapshotfile - help: Restores data from a previous snapshot - takes_value: true - - host: - short: h - required: false - long: host - value_name: host - help: Sets the host to which the server will bind - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the port to which the server will bind - takes_value: true - - noart: - required: false - long: noart - help: Disables terminal artwork - takes_value: false - - nosave: - required: false - long: nosave - help: Disables automated background saving - takes_value: false - - saveduration: - required: false - long: saveduration - value_name: duration - short: S - takes_value: true - help: Set the BGSAVE duration - - snapevery: - required: false - long: snapevery - value_name: duration - help: Set the periodic snapshot duration - takes_value: true - - snapkeep: - required: false - long: snapkeep - value_name: count - help: Sets the number of most recent snapshots to keep - takes_value: true - - sslkey: - required: false - long: sslkey - short: k - value_name: key - help: Sets the PEM key file to use for SSL/TLS - takes_value: true - - sslchain: - required: false - long: sslchain - short: z - value_name: chain - help: Sets the PEM chain file to use for SSL/TLS - takes_value: true - - sslonly: - required: false - long: sslonly - takes_value: false - help: Tells the server to only accept SSL connections and disables the non-SSL port + - config: + short: c + required: false + long: withconfig + value_name: cfgfile + help: Sets a configuration file to start tdb + takes_value: true + - restore: + short: r + required: false + long: restore + value_name: snapshotfile + help: Restores data from a previous snapshot + takes_value: true + - host: + short: h + required: false + long: host + value_name: host + help: Sets the host to which the server will bind + takes_value: true + - port: + short: p + required: false + long: port + value_name: port + help: Sets the port to which the server will bind + takes_value: true + - noart: + required: false + long: noart + help: Disables terminal artwork + takes_value: false + - nosave: + required: false + long: nosave + help: Disables automated background saving + takes_value: false + - saveduration: + required: false + long: saveduration + value_name: duration + short: S + takes_value: true + help: Set the BGSAVE duration + - snapevery: + required: false + long: snapevery + value_name: duration + help: Set the periodic snapshot duration + takes_value: true + - snapkeep: + required: false + long: snapkeep + value_name: count + help: Sets the number of most recent snapshots to keep + takes_value: true + - sslkey: + required: false + long: sslkey + short: k + value_name: key + help: Sets the PEM key file to use for SSL/TLS + takes_value: true + - sslchain: + required: false + long: sslchain + short: z + value_name: chain + help: Sets the PEM chain file to use for SSL/TLS + takes_value: true + - sslonly: + required: false + long: sslonly + takes_value: false + help: >- + Tells the server to only accept SSL connections and disables the non-SSL + port diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index 73932653..2953c7e4 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -461,17 +461,19 @@ pub fn get_config_file_or_return_cfg() -> Result Result Result SnapshotConfig::Disabled, }; - let sslcfg = match (sslkey, sslchain) { - (Some(key), Some(chain)) => { + let sslcfg = match ( + sslkey.map(|val| val.to_owned()), + sslchain.map(|val| val.to_owned()), + ) { + (None, None) => { if sslonly { - SslConfig::EnabledOnly(SslOpts::new(key.to_owned(), chain.to_owned())) + return Err(ConfigError::CliArgErr( + "You mast pass values for both --sslkey and --sslchain to use the --sslonly flag" + )); } else { - SslConfig::Enabled(SslOpts::new(key.to_owned(), chain.to_owned())) + SslConfig::Disabled } } - (None, None) => { + (Some(key), Some(chain)) => { if sslonly { - log::warn!("Ignoring --sslonly flag as SSL wasn't enabled") + SslConfig::EnabledOnly(SslOpts::new(key, chain)) + } else { + SslConfig::Enabled(SslOpts::new(key, chain)) } - SslConfig::Disabled } - (_, _) => { + _ => { return Err(ConfigError::CliArgErr( "To use SSL, pass values for both --sslkey and --sslchain", )); From 68552b4d718cdb0cad996f165cf4ac065f6337f2 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sun, 10 Jan 2021 21:01:12 +0530 Subject: [PATCH 07/17] Enable TLS port to be set and impl `MultiListener` Signed-off-by: Sayan Nandan --- README.md | 2 +- examples/config-files/ssl.toml | 1 + server/src/config/mod.rs | 177 +++++++++++++----------- server/src/dbnet.rs | 242 +++++++++++++++++++++++++++------ server/src/main.rs | 38 ++---- server/src/protocol/tls.rs | 10 +- 6 files changed, 317 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index af5f6a83..75021a6f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ TerrabaseDB (or TDB for short) is an effort to provide the best of key/value sto ## Getting started 🚀 -1. Download a bundle for your platform from [here ⬇️ ](https://github.com/terrabasedb/terrabase/releases) +1. Download a bundle for your platform from [here ⬇️ ](https://github.com/terrabasedb/terrabasedb/releases) 2. Unzip the bundle 3. Make the files executable (run `chmod +x tdb tsh` on *nix systems) 4. First run `tdb` to start the database server and then run `tsh` to start the interactive shell diff --git a/examples/config-files/ssl.toml b/examples/config-files/ssl.toml index 9c74046a..c4e951e3 100644 --- a/examples/config-files/ssl.toml +++ b/examples/config-files/ssl.toml @@ -6,6 +6,7 @@ noart = false [ssl] key = "/path/to/keyfile.pem" chain = "/path/to/chain.pem" +port = 2004 [bgsave] enabled = true diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index 2953c7e4..b0146920 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -34,6 +34,10 @@ use std::path::PathBuf; use tokio::net::ToSocketAddrs; use toml; +const DEFAULT_IPV4: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); +const DEFAULT_PORT: u16 = 2003; +const DEFAULT_SSL_PORT: u16 = 2004; + /// This struct is an _object representation_ used for parsing the TOML file #[derive(Deserialize, Debug, PartialEq)] pub struct Config { @@ -126,38 +130,74 @@ pub struct ConfigKeySnapshot { atmost: usize, } -/// The SSL configuration +/// Port configuration /// -/// This enumeration determines whether the SSL listener is: -/// - `Enabled`: This means that the database server will be listening to both +/// This enumeration determines whether the ports are: +/// - `Multi`: This means that the database server will be listening to both /// SSL **and** non-SSL requests -/// - `EnabledOnly` : This means that the database server will only accept SSL requests +/// - `SecureOnly` : This means that the database server will only accept SSL requests /// and will not even activate the non-SSL socket -/// - `Disabled` : This indicates that the server would only accept non-SSL connections +/// - `InsecureOnly` : This indicates that the server would only accept non-SSL connections /// and will not even activate the SSL socket #[derive(Debug, PartialEq)] -pub enum SslConfig { - Enabled(SslOpts), - EnabledOnly(SslOpts), - Disabled, +pub enum PortConfig { + SecureOnly { + host: IpAddr, + ssl: SslOpts, + }, + Multi { + host: IpAddr, + port: u16, + ssl: SslOpts, + }, + InsecureOnly { + host: IpAddr, + port: u16, + }, +} + +impl PortConfig { + pub const fn default() -> PortConfig { + PortConfig::InsecureOnly { + host: DEFAULT_IPV4, + port: DEFAULT_PORT, + } + } +} + +impl PortConfig { + pub const fn new_secure_only(host: IpAddr, ssl: SslOpts) -> Self { + PortConfig::SecureOnly { host, ssl } + } + pub const fn new_insecure_only(host: IpAddr, port: u16) -> Self { + PortConfig::InsecureOnly { host, port } + } + pub const fn new_multi(host: IpAddr, port: u16, ssl: SslOpts) -> Self { + PortConfig::Multi { host, port, ssl } + } } #[derive(Deserialize, Debug, PartialEq)] pub struct KeySslOpts { key: String, chain: String, + port: u16, only: Option, } #[derive(Deserialize, Debug, PartialEq)] pub struct SslOpts { - key: String, - chain: String, + pub key: String, + pub chain: String, + pub port: u16, } impl SslOpts { - pub const fn new(key: String, chain: String) -> Self { - SslOpts { key, chain } + pub const fn new(key: String, chain: String, port: u16) -> Self { + SslOpts { key, chain, port } + } + pub fn get_host_port_tuple(&self, host: String) -> impl ToSocketAddrs { + ((host), self.port) } } @@ -217,18 +257,14 @@ impl SnapshotConfig { /// configuration #[derive(Debug, PartialEq)] pub struct ParsedConfig { - /// A valid IPv4/IPv6 address - host: IpAddr, - /// A valid port - port: u16, /// If `noart` is set to true, no terminal artwork should be displayed noart: bool, /// The BGSAVE configuration pub bgsave: BGSave, /// The snapshot configuration pub snapshot: SnapshotConfig, - /// The SSL configuration - pub ssl: SslConfig, + /// Port configuration + pub ports: PortConfig, } impl ParsedConfig { @@ -247,8 +283,6 @@ impl ParsedConfig { /// TOML file (represented as an object) fn from_config(cfg_info: Config) -> Self { ParsedConfig { - host: cfg_info.server.host, - port: cfg_info.server.port, noart: if let Some(noart) = cfg_info.server.noart { noart } else { @@ -270,20 +304,33 @@ impl ParsedConfig { } else { SnapshotConfig::default() }, - ssl: if let Some(sslopts) = cfg_info.ssl { + ports: if let Some(sslopts) = cfg_info.ssl { if sslopts.only.is_some() { - SslConfig::EnabledOnly(SslOpts { - key: sslopts.key, - chain: sslopts.chain, - }) + PortConfig::SecureOnly { + ssl: SslOpts { + key: sslopts.key, + chain: sslopts.chain, + port: sslopts.port, + }, + + host: cfg_info.server.host, + } } else { - SslConfig::Enabled(SslOpts { - key: sslopts.key, - chain: sslopts.chain, - }) + PortConfig::Multi { + ssl: SslOpts { + key: sslopts.key, + chain: sslopts.chain, + port: sslopts.port, + }, + host: cfg_info.server.host, + port: cfg_info.server.port, + } } } else { - SslConfig::Disabled + PortConfig::InsecureOnly { + host: cfg_info.server.host, + port: cfg_info.server.port, + } }, } } @@ -296,42 +343,37 @@ impl ParsedConfig { /// and a supplied `port` pub const fn default_with_port(port: u16) -> Self { ParsedConfig { - host: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), - port, noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), - ssl: SslConfig::Disabled, + ports: PortConfig::new_insecure_only(DEFAULT_IPV4, port), } } + pub const fn default_ports() -> PortConfig { + PortConfig::default() + } /// Create a new `ParsedConfig` with the default `port` and `noart` settngs /// and a supplied `host` pub const fn default_with_host(host: IpAddr) -> Self { ParsedConfig::new( - host, - 2003, false, BGSave::default(), SnapshotConfig::default(), - SslConfig::Disabled, + PortConfig::new_insecure_only(host, 2003), ) } /// Create a new `ParsedConfig` with all the fields pub const fn new( - host: IpAddr, - port: u16, noart: bool, bgsave: BGSave, snapshot: SnapshotConfig, - ssl: SslConfig, + ports: PortConfig, ) -> Self { ParsedConfig { - host, - port, noart, bgsave, snapshot, - ssl, + ports, } } /// Create a default `ParsedConfig` with the following setup defaults: @@ -343,18 +385,12 @@ impl ParsedConfig { /// - `ssl` : disabled pub const fn default() -> Self { ParsedConfig { - host: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), - port: 2003, noart: false, bgsave: BGSave::default(), snapshot: SnapshotConfig::default(), - ssl: SslConfig::Disabled, + ports: PortConfig::new_insecure_only(DEFAULT_IPV4, 2003), } } - /// Return a (host, port) tuple which can be bound to with `TcpListener` - pub fn get_host_port_tuple(&self) -> impl ToSocketAddrs { - ((self.host), self.port) - } /// Returns `false` if `noart` is enabled. Otherwise it returns `true` pub const fn is_artful(&self) -> bool { !self.noart @@ -574,7 +610,7 @@ pub fn get_config_file_or_return_cfg() -> Result SnapshotConfig::Disabled, }; - let sslcfg = match ( + let portcfg = match ( sslkey.map(|val| val.to_owned()), sslchain.map(|val| val.to_owned()), ) { @@ -584,14 +620,14 @@ pub fn get_config_file_or_return_cfg() -> Result { if sslonly { - SslConfig::EnabledOnly(SslOpts::new(key, chain)) + PortConfig::new_secure_only(host, SslOpts::new(key, chain, DEFAULT_SSL_PORT)) } else { - SslConfig::Enabled(SslOpts::new(key, chain)) + PortConfig::new_multi(host, port, SslOpts::new(key, chain, DEFAULT_SSL_PORT)) } } _ => { @@ -600,7 +636,7 @@ pub fn get_config_file_or_return_cfg() -> Result, + db: CoreDB, + signal: broadcast::Sender<()>, + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + ) -> Self { + let listener = TcpListener::bind((host, port)) + .await + .expect("Failed to bind to port"); + MultiListener::InsecureOnly(Listener { + listener, + db, + climit, + signal, + terminate_tx, + terminate_rx, + }) + } + pub async fn new_secure_only( + host: IpAddr, + climit: Arc, + db: CoreDB, + signal: broadcast::Sender<()>, + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + ssl: SslOpts, + ) -> Self { + let listener = TcpListener::bind((host, ssl.port)) + .await + .expect("Failed to bind to port"); + MultiListener::SecureOnly( + SslListener::new_pem_based_ssl_connection( + ssl.key, + ssl.chain, + db, + listener, + climit, + signal, + terminate_tx, + terminate_rx, + ) + .expect("Couldn't bind to secure port"), + ) + } + pub async fn run_server(&mut self) -> TResult<()> { + match self { + MultiListener::SecureOnly(secure_listener) => secure_listener.run().await, + MultiListener::InsecureOnly(insecure_listener) => insecure_listener.run().await, + MultiListener::Multi(insecure_listener, secure_listener) => { + secure_listener.run().await?; + insecure_listener.run().await + } + } + } + pub fn print_binding(&self) { + match self { + MultiListener::SecureOnly(secure_listener) => { + log::info!( + "Server started on tps://{}", + secure_listener.listener.local_addr().expect("Failed to g") + ) + } + MultiListener::InsecureOnly(insecure_listener) => { + log::info!( + "Server started on tp://{}", + insecure_listener + .listener + .local_addr() + .expect("Failed to g") + ) + } + MultiListener::Multi(insecure_listener, secure_listener) => { + log::info!( + "Listening to tp://{} and tps://{}", + insecure_listener + .listener + .local_addr() + .expect("Failed to g"), + secure_listener.listener.local_addr().expect("Failed to g") + ) + } + } + } + pub async fn finish_with_termsig(self) { + match self { + MultiListener::InsecureOnly(server) => { + let Listener { + mut terminate_rx, + terminate_tx, + signal, + db, + .. + } = server; + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + () + } else { + log::error!("Failed to flush data to disk"); + loop { + // Keep looping until we successfully write the in-memory table to disk + log::warn!("Press enter to try again..."); + io::stdout().flush().unwrap(); + io::stdin().read(&mut [0]).unwrap(); + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + break; + } else { + continue; + } + } + } + drop(signal); + drop(terminate_tx); + let _ = terminate_rx.recv().await; + } + MultiListener::SecureOnly(server) => { + let SslListener { + mut terminate_rx, + terminate_tx, + signal, + db, + .. + } = server; + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + () + } else { + log::error!("Failed to flush data to disk"); + loop { + // Keep looping until we successfully write the in-memory table to disk + log::warn!("Press enter to try again..."); + io::stdout().flush().unwrap(); + io::stdin().read(&mut [0]).unwrap(); + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + break; + } else { + continue; + } + } + } + drop(signal); + drop(terminate_tx); + let _ = terminate_rx.recv().await; + } + _ => { + todo!("Multiple listeners haven't been implemented yet!"); + } + } + } +} + /// Start the server waiting for incoming connections or a CTRL+C signal pub async fn run( - listener: TcpListener, + ports: PortConfig, bgsave_cfg: BGSave, snapshot_cfg: SnapshotConfig, sig: impl Future, @@ -262,54 +428,44 @@ pub async fn run( } }, } - log::info!( - "Started server on terrapipe://{}", - listener - .local_addr() - .expect("The local address couldn't be fetched. Please file a bug report") - ); - let mut server = Listener { - listener, - db, - climit: Arc::new(Semaphore::new(50000)), - signal, - terminate_tx, - terminate_rx, + let climit = Arc::new(Semaphore::new(50000)); + let mut server = match ports { + PortConfig::InsecureOnly { host, port } => { + MultiListener::new_insecure_only( + host, + port, + climit.clone(), + db, + signal, + terminate_tx, + terminate_rx, + ) + .await + } + PortConfig::SecureOnly { host, ssl } => { + MultiListener::new_secure_only( + host, + climit.clone(), + db, + signal, + terminate_tx, + terminate_rx, + ssl, + ) + .await + } + _ => { + todo!("Multiple listeners haven't been implemented yet!") + } }; + server.print_binding(); tokio::select! { - _ = server.run() => {} + _ = server.run_server() => {} _ = sig => { log::info!("Signalling all workers to shut down"); } } - let Listener { - mut terminate_rx, - terminate_tx, - signal, - db, - .. - } = server; - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - () - } else { - log::error!("Failed to flush data to disk"); - loop { - // Keep looping until we successfully write the in-memory table to disk - log::warn!("Press enter to try again..."); - io::stdout().flush().unwrap(); - io::stdin().read(&mut [0]).unwrap(); - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - break; - } else { - continue; - } - } - } - drop(signal); - drop(terminate_tx); - let _ = terminate_rx.recv().await; + server.finish_with_termsig().await; terminal::write_info("Goodbye :)\n").unwrap(); } diff --git a/server/src/main.rs b/server/src/main.rs index 9389552a..e71ad47b 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -26,8 +26,8 @@ //! the modules for their respective documentation. use crate::config::BGSave; +use crate::config::PortConfig; use crate::config::SnapshotConfig; -use tokio::net::TcpListener; mod config; use std::env; mod admin; @@ -54,7 +54,7 @@ use jemallocator::Jemalloc; static GLOBAL: Jemalloc = Jemalloc; /// The version text -static MSG: &'static str = "TerrabaseDB v0.5.0 | https://github.com/terrabasedb/terrabase"; +static MSG: &'static str = "TerrabaseDB v0.5.0 | https://github.com/terrabasedb/terrabasedb"; /// The terminal art for `!noart` configurations static TEXT: &'static str = " _______ _ _____ ____ @@ -76,7 +76,7 @@ async fn main() { // Start the server which asynchronously waits for a CTRL+C signal // which will safely shut down the server let (tcplistener, bgsave_config, snapshot_config, restore_filepath) = - check_args_or_connect().await; + check_args_and_get_cfg().await; run( tcplistener, bgsave_config, @@ -87,10 +87,10 @@ async fn main() { .await; } -/// This function checks the command line arguments and binds to an appropriate -/// port and host, as per the supplied configuration options -async fn check_args_or_connect() -> ( - TcpListener, +/// This function checks the command line arguments and either returns a config object +/// or prints an error to `stderr` and terminates the server +async fn check_args_and_get_cfg() -> ( + PortConfig, BGSave, SnapshotConfig, Option, @@ -104,35 +104,17 @@ async fn check_args_or_connect() -> ( println!("{}", MSG); } log::info!("Using settings from supplied configuration"); - ( - TcpListener::bind(cfg.get_host_port_tuple()).await, - cfg.bgsave, - cfg.snapshot, - file, - ) + (cfg.ports, cfg.bgsave, cfg.snapshot, file) } Ok(config::ConfigType::Def(cfg, file)) => { println!("{}\n{}", TEXT, MSG); log::warn!("No configuration file supplied. Using default settings"); - ( - TcpListener::bind(cfg.get_host_port_tuple()).await, - cfg.bgsave, - cfg.snapshot, - file, - ) + (cfg.ports, cfg.bgsave, cfg.snapshot, file) } Err(e) => { log::error!("{}", e); std::process::exit(0x100); } }; - match binding_and_cfg { - (Ok(b), bgsave_cfg, snapshot_cfg, restore_file) => { - (b, bgsave_cfg, snapshot_cfg, restore_file) - } - (Err(e), _, _, _) => { - log::error!("Failed to bind to socket with error: '{}'", e); - std::process::exit(0x100); - } - } + binding_and_cfg } diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 7d8deca1..069370d9 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -45,17 +45,17 @@ use tokio_openssl::SslStream; pub struct SslListener { /// An atomic reference to the coretable - db: CoreDB, + pub db: CoreDB, /// The incoming connection listener (binding) - listener: TcpListener, + pub listener: TcpListener, /// The maximum number of connections climit: Arc, /// The shutdown broadcaster - signal: broadcast::Sender<()>, + pub signal: broadcast::Sender<()>, // When all `Sender`s are dropped - the `Receiver` gets a `None` value // We send a clone of `terminate_tx` to each `CHandler` - terminate_tx: mpsc::Sender<()>, - terminate_rx: mpsc::Receiver<()>, + pub terminate_tx: mpsc::Sender<()>, + pub terminate_rx: mpsc::Receiver<()>, acceptor: Arc, } From 17860ef45aeb77948d2685bfb0f403be610f0089 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sun, 10 Jan 2021 22:16:57 +0530 Subject: [PATCH 08/17] Implement the `Multi` variant for `MultiListener` Signed-off-by: Sayan Nandan --- server/src/dbnet.rs | 135 ++++++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 43 deletions(-) diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index 61754317..7aafe2db 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -293,6 +293,45 @@ impl MultiListener { .expect("Couldn't bind to secure port"), ) } + pub async fn new_multi( + host: IpAddr, + port: u16, + climit: Arc, + db: CoreDB, + signal: broadcast::Sender<()>, + terminate_tx: mpsc::Sender<()>, + terminate_rx: mpsc::Receiver<()>, + ssl_terminate_tx: mpsc::Sender<()>, + ssl_terminate_rx: mpsc::Receiver<()>, + ssl: SslOpts, + ) -> Self { + let listener = TcpListener::bind((host, ssl.port)) + .await + .expect("Failed to bind to port"); + let secure_listener = SslListener::new_pem_based_ssl_connection( + ssl.key, + ssl.chain, + db.clone(), + listener, + climit.clone(), + signal.clone(), + ssl_terminate_tx, + ssl_terminate_rx, + ) + .expect("Couldn't bind to secure port"); + let listener = TcpListener::bind((host, port)) + .await + .expect("Failed to bind to port"); + let insecure_listener = Listener { + listener, + db, + climit, + signal, + terminate_tx, + terminate_rx, + }; + MultiListener::Multi(insecure_listener, secure_listener) + } pub async fn run_server(&mut self) -> TResult<()> { match self { MultiListener::SecureOnly(secure_listener) => secure_listener.run().await, @@ -339,27 +378,8 @@ impl MultiListener { mut terminate_rx, terminate_tx, signal, - db, .. } = server; - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - () - } else { - log::error!("Failed to flush data to disk"); - loop { - // Keep looping until we successfully write the in-memory table to disk - log::warn!("Press enter to try again..."); - io::stdout().flush().unwrap(); - io::stdin().read(&mut [0]).unwrap(); - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - break; - } else { - continue; - } - } - } drop(signal); drop(terminate_tx); let _ = terminate_rx.recv().await; @@ -369,32 +389,29 @@ impl MultiListener { mut terminate_rx, terminate_tx, signal, - db, .. } = server; - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - () - } else { - log::error!("Failed to flush data to disk"); - loop { - // Keep looping until we successfully write the in-memory table to disk - log::warn!("Press enter to try again..."); - io::stdout().flush().unwrap(); - io::stdin().read(&mut [0]).unwrap(); - if let Ok(_) = db.flush_db() { - log::info!("Successfully saved data to disk"); - break; - } else { - continue; - } - } - } drop(signal); drop(terminate_tx); let _ = terminate_rx.recv().await; } - _ => { + MultiListener::Multi(insecure, secure) => { + let Listener { + mut terminate_rx, + terminate_tx, + signal, + .. + } = insecure; + drop((signal, terminate_tx)); + let _ = terminate_rx.recv().await; + let SslListener { + mut terminate_rx, + terminate_tx, + signal, + .. + } = secure; + drop((signal, terminate_tx)); + let _ = terminate_rx.recv().await; todo!("Multiple listeners haven't been implemented yet!"); } } @@ -435,7 +452,7 @@ pub async fn run( host, port, climit.clone(), - db, + db.clone(), signal, terminate_tx, terminate_rx, @@ -446,7 +463,7 @@ pub async fn run( MultiListener::new_secure_only( host, climit.clone(), - db, + db.clone(), signal, terminate_tx, terminate_rx, @@ -454,8 +471,22 @@ pub async fn run( ) .await } - _ => { - todo!("Multiple listeners haven't been implemented yet!") + PortConfig::Multi { host, port, ssl } => { + let (ssl_terminate_tx, ssl_terminate_rx) = mpsc::channel::<()>(1); + let server = MultiListener::new_multi( + host, + port, + climit, + db.clone(), + signal, + terminate_tx, + terminate_rx, + ssl_terminate_tx, + ssl_terminate_rx, + ssl, + ) + .await; + server } }; server.print_binding(); @@ -466,6 +497,24 @@ pub async fn run( } } server.finish_with_termsig().await; + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + () + } else { + log::error!("Failed to flush data to disk"); + loop { + // Keep looping until we successfully write the in-memory table to disk + log::warn!("Press enter to try again..."); + io::stdout().flush().unwrap(); + io::stdin().read(&mut [0]).unwrap(); + if let Ok(_) = db.flush_db() { + log::info!("Successfully saved data to disk"); + break; + } else { + continue; + } + } + } terminal::write_info("Goodbye :)\n").unwrap(); } From 30c9f2991bc109201d5dbd1dc2866acfa599eaf7 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sun, 10 Jan 2021 22:29:04 +0530 Subject: [PATCH 09/17] Enable query execution for secure connections This commit enables queries to be executed on secure connections. At the same time, the `execute_query_ssl` function was removed as `execute_query` has been modified so that it can be used by both secure and insecure connections Signed-off-by: Sayan Nandan --- server/src/config/mod.rs | 2 ++ server/src/coredb/mod.rs | 7 +----- server/src/protocol/tls.rs | 44 +++++++++++--------------------------- 3 files changed, 16 insertions(+), 37 deletions(-) diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index b0146920..5e9c0f4a 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -157,6 +157,7 @@ pub enum PortConfig { } impl PortConfig { + #[cfg(test)] pub const fn default() -> PortConfig { PortConfig::InsecureOnly { host: DEFAULT_IPV4, @@ -349,6 +350,7 @@ impl ParsedConfig { ports: PortConfig::new_insecure_only(DEFAULT_IPV4, port), } } + #[cfg(test)] pub const fn default_ports() -> PortConfig { PortConfig::default() } diff --git a/server/src/coredb/mod.rs b/server/src/coredb/mod.rs index 2f5e5645..095b6dcf 100644 --- a/server/src/coredb/mod.rs +++ b/server/src/coredb/mod.rs @@ -21,10 +21,10 @@ //! # The core database engine -use crate::dbnet::Con; use crate::config::BGSave; use crate::config::SnapshotConfig; use crate::config::SnapshotPref; +use crate::dbnet::Con; use crate::diskstore; use crate::protocol::tls::SslConnection; use crate::protocol::{Connection, Query}; @@ -258,11 +258,6 @@ impl CoreDB { } } - pub async fn execute_query_ssl(&self, _query: Query, _con: &mut SslConnection) -> TResult<()> { - // TODO(@ohsayan): Implement SSL-connection queries - todo!() - } - /// Create a new `CoreDB` instance /// /// This also checks if a local backup of previously saved data is available. diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 069370d9..9b7975f8 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -24,6 +24,7 @@ use super::responses; use super::IoResult; use super::ParseResult; use super::QueryResult; +use crate::dbnet::Con; use crate::dbnet::Terminator; use crate::resp::Writable; use crate::CoreDB; @@ -149,7 +150,11 @@ impl SslConnectionHandler { } }; match try_df { - Ok(QueryResult::Q(s)) => self.db.execute_query_ssl(s, &mut self.con).await?, + Ok(QueryResult::Q(s)) => { + self.db + .execute_query(s, &mut Con::Secure(&mut self.con)) + .await? + } Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, Ok(QueryResult::Empty) => return Ok(()), Err(e) => return Err(e.into()), @@ -158,6 +163,13 @@ impl SslConnectionHandler { Ok(()) } } +impl Drop for SslConnectionHandler { + fn drop(&mut self) { + // Make sure that the permit is returned to the semaphore + // in the case that there is a panic inside + self.climit.add_permits(1); + } +} pub struct SslConnection { stream: BufWriter>, @@ -241,33 +253,3 @@ impl SslConnection { Ok(()) } } - -/// A per-connection handler -pub struct SslCHandler { - db: CoreDB, - con: SslConnection, - climit: Arc, - terminator: Terminator, - _term_sig_tx: mpsc::Sender<()>, -} - -impl SslCHandler { - /// Process the incoming connection - pub async fn run(&mut self) -> TResult<()> { - while !self.terminator.is_termination_signal() { - let try_df = tokio::select! { - tdf = self.con.read_query() => tdf, - _ = self.terminator.receive_signal() => { - return Ok(()); - } - }; - match try_df { - Ok(QueryResult::Q(s)) => self.db.execute_query_ssl(s, &mut self.con).await?, - Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, - Ok(QueryResult::Empty) => return Ok(()), - Err(e) => return Err(e.into()), - } - } - Ok(()) - } -} From 053be8c44a5e984327ef0cbafc491bf2db3ade43 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Mon, 11 Jan 2021 12:20:07 +0530 Subject: [PATCH 10/17] Enable parallel handling of secure/insecure ports Also, the imports were optimized. Signed-off-by: Sayan Nandan --- server/src/coredb/mod.rs | 3 +-- server/src/dbnet.rs | 30 ++++++++++++++++++++++++++---- server/src/protocol/tls.rs | 2 +- server/src/queryengine/mod.rs | 2 +- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/server/src/coredb/mod.rs b/server/src/coredb/mod.rs index 095b6dcf..3d4d5068 100644 --- a/server/src/coredb/mod.rs +++ b/server/src/coredb/mod.rs @@ -26,8 +26,7 @@ use crate::config::SnapshotConfig; use crate::config::SnapshotPref; use crate::dbnet::Con; use crate::diskstore; -use crate::protocol::tls::SslConnection; -use crate::protocol::{Connection, Query}; +use crate::protocol::Query; use crate::queryengine; use bytes::Bytes; use diskstore::PERSIST_FILE; diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index 7aafe2db..c049ac8f 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -183,7 +183,7 @@ impl<'a> Con<'a> { Con::Insecure(con) } /// Create a new **encrypted** connection instance - pub fn init_ssl<'b>(con: &'b mut SslConnection) -> Self + pub fn init_secure<'b>(con: &'b mut SslConnection) -> Self where 'b: 'a, { @@ -218,7 +218,7 @@ impl CHandler { match try_df { Ok(Q(s)) => { self.db - .execute_query(s, &mut Con::Insecure(&mut self.con)) + .execute_query(s, &mut Con::init(&mut self.con)) .await? } Ok(E(r)) => self.con.close_conn_with_error(r).await?, @@ -239,6 +239,16 @@ impl Drop for CHandler { } use std::io::{self, prelude::*}; +/// Multiple Listener Interface +/// +/// A `MultiListener` is an abstraction over an `SslListener` or a `Listener` to facilitate +/// easier asynchronous listening on multiple ports. +/// +/// - The `SecureOnly` variant holds an `SslListener` +/// - The `InsecureOnly` variant holds a `Listener` +/// - The `Multi` variant holds both an `SslListener` and a `Listener` +/// This variant enables listening to both secure and insecure sockets at the same time +/// asynchronously enum MultiListener { SecureOnly(SslListener), InsecureOnly(Listener), @@ -246,6 +256,7 @@ enum MultiListener { } impl MultiListener { + /// Create a new `InsecureOnly` listener pub async fn new_insecure_only( host: IpAddr, port: u16, @@ -267,6 +278,7 @@ impl MultiListener { terminate_rx, }) } + /// Create a new `SecureOnly` listener pub async fn new_secure_only( host: IpAddr, climit: Arc, @@ -293,6 +305,7 @@ impl MultiListener { .expect("Couldn't bind to secure port"), ) } + /// Create a new `Multi` listener that has both a secure and an insecure listener pub async fn new_multi( host: IpAddr, port: u16, @@ -332,16 +345,22 @@ impl MultiListener { }; MultiListener::Multi(insecure_listener, secure_listener) } + /// Start the server + /// + /// The running of single and/or parallel listeners is handled by this function by + /// exploiting the working of async functions pub async fn run_server(&mut self) -> TResult<()> { match self { MultiListener::SecureOnly(secure_listener) => secure_listener.run().await, MultiListener::InsecureOnly(insecure_listener) => insecure_listener.run().await, MultiListener::Multi(insecure_listener, secure_listener) => { + insecure_listener.run().await?; secure_listener.run().await?; - insecure_listener.run().await + Ok(()) } } } + /// Print the port binding status pub fn print_binding(&self) { match self { MultiListener::SecureOnly(secure_listener) => { @@ -371,6 +390,10 @@ impl MultiListener { } } } + /// Signal the ports to shut down and only return after they have shut down + /// + /// **Do note:** This function doesn't flush the `CoreDB` object! The **caller has to + /// make sure that the data is saved!** pub async fn finish_with_termsig(self) { match self { MultiListener::InsecureOnly(server) => { @@ -412,7 +435,6 @@ impl MultiListener { } = secure; drop((signal, terminate_tx)); let _ = terminate_rx.recv().await; - todo!("Multiple listeners haven't been implemented yet!"); } } } diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 9b7975f8..e3e2bf57 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -152,7 +152,7 @@ impl SslConnectionHandler { match try_df { Ok(QueryResult::Q(s)) => { self.db - .execute_query(s, &mut Con::Secure(&mut self.con)) + .execute_query(s, &mut Con::init_secure(&mut self.con)) .await? } Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, diff --git a/server/src/queryengine/mod.rs b/server/src/queryengine/mod.rs index df5903df..0299ddf2 100644 --- a/server/src/queryengine/mod.rs +++ b/server/src/queryengine/mod.rs @@ -24,7 +24,7 @@ use crate::coredb::CoreDB; use crate::dbnet::Con; use crate::protocol::ActionGroup; -use crate::protocol::{responses, Connection}; +use crate::protocol::{responses}; use crate::{admin, kvengine}; use libtdb::TResult; mod tags { From 07b99857329b650ec7afa10478d993bbdaaab8fb Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Mon, 11 Jan 2021 13:40:29 +0530 Subject: [PATCH 11/17] Add client side SSL support Signed-off-by: Sayan Nandan --- Cargo.lock | 216 +++++++++++++++++++++++++++++++++---- cli/Cargo.toml | 10 +- cli/src/argparse.rs | 35 +++--- cli/src/cli.yml | 49 +++++---- cli/src/protocol/mod.rs | 123 +++++++++++++++++++-- server/src/config/mod.rs | 3 - server/src/protocol/tls.rs | 2 + server/src/resp/mod.rs | 2 - 8 files changed, 369 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f1f8d1c..aaedab6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,6 +69,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" +[[package]] +name = "bytes" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72" + [[package]] name = "cc" version = "1.0.59" @@ -165,11 +171,100 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +[[package]] +name = "futures" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c70be434c505aee38639abccb918163b63158a4b4bb791b45b7023044bdc3c9c" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f01c61843314e95f96cc9245702248733a3a3d744e43e2e755e3c7af8348a0a9" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +checksum = "db8d3b0917ff63a2a96173133c02818fac4a746b0a57569d3baca9ec0e945e08" + +[[package]] +name = "futures-executor" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ee9ca2f7eb4475772cf39dd1cd06208dce2670ad38f4d9c7262b3e15f127068" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37c1a51b037b80922864b8eed90692c5cd8abd4c71ce49b77146caa47f3253b" + +[[package]] +name = "futures-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8719ca0e1f3c5e34f3efe4570ef2c0610ca6da85ae7990d472e9cbfba13664" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6adabac1290109cfa089f79192fb6244ad2c3f1cc2281f3e1dd987592b71feb" + +[[package]] +name = "futures-task" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92a0843a2ff66823a8f7c77bffe9a09be2b64e533562c412d63075643ec0038" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] [[package]] name = "getrandom" @@ -246,7 +341,7 @@ checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" name = "libtdb" version = "0.5.0" dependencies = [ - "bytes", + "bytes 0.6.0", "lazy_static", "termcolor", ] @@ -344,9 +439,9 @@ checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" [[package]] name = "openssl" -version = "0.10.31" +version = "0.10.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d008f51b1acffa0d3450a68606e6a51c123012edaacb0f4e1426bd978869187" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -367,9 +462,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.59" +version = "0.9.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de52d8eabd217311538a39bba130d7dea1f1e118010fee7a033d966845e7d5fe" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" dependencies = [ "autocfg", "cc", @@ -405,12 +500,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "pin-project" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.19" @@ -423,6 +544,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + [[package]] name = "proc-macro2" version = "1.0.24" @@ -490,9 +623,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "regex" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -502,9 +635,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "ryu" @@ -605,7 +738,7 @@ name = "tdb" version = "0.5.0" dependencies = [ "bincode", - "bytes", + "bytes 0.6.0", "chrono", "clap", "env_logger", @@ -620,8 +753,8 @@ dependencies = [ "serde", "serde_derive", "tdb_macros", - "tokio", - "tokio-openssl", + "tokio 0.3.6", + "tokio-openssl 0.5.0", "toml", ] @@ -692,7 +825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "720ba21c25078711bf456d607987d95bce90f7c3bea5abe1db587862e7a1e87c" dependencies = [ "autocfg", - "bytes", + "bytes 0.6.0", "futures-core", "libc", "memchr", @@ -703,7 +836,27 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "tokio-macros", + "tokio-macros 0.3.1", + "winapi", +] + +[[package]] +name = "tokio" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd" +dependencies = [ + "autocfg", + "bytes 1.0.0", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros 1.0.0", "winapi", ] @@ -718,6 +871,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42517d2975ca3114b22a16192634e8241dc5cc1f130be194645970cc1c371494" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-openssl" version = "0.5.0" @@ -726,7 +890,19 @@ checksum = "d01e5cc2d3a154fa310982d0affaec8140d6476805422265b2f648eb813f937f" dependencies = [ "openssl", "openssl-sys", - "tokio", + "tokio 0.3.6", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1bec5c0a4aa71e3459802c7a12e8912c2091ce2151004f9ce95cc5d1c6124e" +dependencies = [ + "futures", + "openssl", + "pin-project", + "tokio 1.0.1", ] [[package]] @@ -742,12 +918,14 @@ dependencies = [ name = "tsh" version = "0.5.0" dependencies = [ - "bytes", + "bytes 1.0.0", "clap", "lazy_static", "libtdb", + "openssl", "regex", - "tokio", + "tokio 1.0.1", + "tokio-openssl 0.6.1", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2c436129..3a15caec 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -8,8 +8,10 @@ edition = "2018" [dependencies] libtdb = {path = "../libtdb"} -tokio = {version = "0.3.6", features = ["full"]} -bytes = "0.6.0" -regex = "1.4.2" +tokio = {version = "1.0.1", features = ["full"]} +bytes = "1.0.0" +regex = "1.4.3" lazy_static = "1.4.0" -clap = {version = "2.33.3", features=["yaml"]} \ No newline at end of file +clap = {version = "2.33.3", features=["yaml"]} +openssl = { version = "0.10.32", features = ["vendored"] } +tokio-openssl = "0.6.1" \ No newline at end of file diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs index ca547e19..568a3511 100644 --- a/cli/src/argparse.rs +++ b/cli/src/argparse.rs @@ -23,6 +23,7 @@ use crate::protocol; use clap::load_yaml; use clap::App; use libtdb::terrapipe::ADDR; +use protocol::{Con, Connection, SslConnection}; use std::io::{self, prelude::*}; use std::process; const MSG_WELCOME: &'static str = "TerrabaseDB v0.5.0"; @@ -39,6 +40,7 @@ pub async fn start_repl() { Some(h) => h.to_owned(), None => ADDR.to_owned(), }; + let domain = host.clone(); host.push(':'); match matches.value_of("port") { Some(p) => match p.parse::() { @@ -50,23 +52,32 @@ pub async fn start_repl() { }, None => host.push_str("2003"), } + let mut con = if let Some(sslcert) = matches.value_of("cert") { + let con = match SslConnection::new(&domain, &host, sslcert).await { + Ok(c) => c, + Err(e) => { + eprintln!("ERROR: {}", e); + process::exit(0x100); + } + }; + Con::Secure(con) + } else { + let con = match Connection::new(&host).await { + Ok(c) => c, + Err(e) => { + eprintln!("ERROR: {}", e); + process::exit(0x100); + } + }; + Con::Insecure(con) + }; if let Some(eval_expr) = matches.value_of("eval") { if eval_expr.len() == 0 { return; } - if let Err(e) = protocol::Connection::oneshot(&host, eval_expr.to_string()).await { - eprintln!("ERROR: {}", e); - process::exit(0x100); - } + con.execute_query(eval_expr.to_string()).await; return; } - let mut con = match protocol::Connection::new(&host).await { - Ok(c) => c, - Err(e) => { - eprintln!("ERROR: {}", e); - process::exit(0x100); - } - }; println!("{}", MSG_WELCOME); loop { print!("tsh>"); @@ -85,6 +96,6 @@ pub async fn start_repl() { // The query was empty, so let it be continue; } - con.run_query(rl).await; + con.execute_query(rl).await; } } diff --git a/cli/src/cli.yml b/cli/src/cli.yml index 0745b293..bd6ae5fb 100644 --- a/cli/src/cli.yml +++ b/cli/src/cli.yml @@ -24,24 +24,31 @@ version: 0.5.0 author: Sayan N. about: The TerrabaseDB Shell (tsh) args: - - host: - short: h - required: false - long: host - value_name: host - help: Sets the remote host to connect to - takes_value: true - - port: - short: p - required: false - long: port - value_name: port - help: Sets the remote port to connect to - takes_value: true - - eval: - short: e - required: false - long: eval - value_name: expression - help: Run an expression without REPL - takes_value: true + - host: + short: h + required: false + long: host + value_name: host + help: Sets the remote host to connect to + takes_value: true + - port: + short: p + required: false + long: port + value_name: port + help: Sets the remote port to connect to + takes_value: true + - eval: + short: e + required: false + long: eval + value_name: expression + help: Run an expression without REPL + takes_value: true + - cert: + short: C + required: false + long: sslcert + value_name: cert + help: Sets the PEM certificate to use for SSL connections + takes_value: true diff --git a/cli/src/protocol/mod.rs b/cli/src/protocol/mod.rs index dde8edb8..280f0423 100644 --- a/cli/src/protocol/mod.rs +++ b/cli/src/protocol/mod.rs @@ -26,14 +26,32 @@ use lazy_static::lazy_static; use libtdb::terrapipe; use libtdb::TResult; use libtdb::BUF_CAP; +use openssl::ssl::SslConnector; +use openssl::ssl::SslMethod; use regex::Regex; +use std::pin::Pin; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use tokio_openssl::SslStream; lazy_static! { static ref RE: Regex = Regex::new("[^\\s\"']+|\"[^\"]*\"|'[^']*'").unwrap(); } +pub enum Con { + Secure(SslConnection), + Insecure(Connection), +} + +impl Con { + pub async fn execute_query(&mut self, query: String) { + match self { + Con::Insecure(con) => con.run_query(query).await, + Con::Secure(con) => con.run_query(query).await, + } + } +} + /// A `Connection` is a wrapper around a`TcpStream` and a read buffer pub struct Connection { stream: TcpStream, @@ -44,21 +62,12 @@ impl Connection { /// Create a new connection, creating a connection to `host` pub async fn new(host: &str) -> TResult { let stream = TcpStream::connect(host).await?; - println!("Connected to {}", host); + println!("Connected to tp://{}", host); Ok(Connection { stream, buffer: BytesMut::with_capacity(BUF_CAP), }) } - pub async fn oneshot(host: &str, query: String) -> TResult<()> { - let mut con = Connection { - stream: TcpStream::connect(host).await?, - buffer: BytesMut::with_capacity(BUF_CAP), - }; - con.run_query(query).await; - drop(con); - Ok(()) - } /// This function will write a query to the stream and read the response from the /// server. It will then determine if the returned response is complete or incomplete /// or invalid. @@ -85,6 +94,100 @@ impl Connection { return; } } + println!( + "The server returned: {}", + String::from_utf8_lossy(&self.buffer) + ); + match self.try_response().await { + ClientResult::Empty(f) => { + self.buffer.advance(f); + eprintln!("ERROR: The remote end reset the connection"); + return; + } + ClientResult::Incomplete => { + continue; + } + ClientResult::Response(r, f) => { + self.buffer.advance(f); + if r.len() == 0 { + return; + } + for group in r { + println!("{}", group); + } + return; + } + ClientResult::InvalidResponse(_) => { + self.buffer.clear(); + eprintln!("{}", ClientResult::InvalidResponse(0)); + return; + } + } + } + } + /// This function is a subroutine of `run_query` used to parse the response packet + async fn try_response(&mut self) -> ClientResult { + if self.buffer.is_empty() { + // The connection was possibly reset + return ClientResult::Empty(0); + } + deserializer::parse(&self.buffer) + } +} + +/// An `SslConnection` is a wrapper around a `SslStream` provided by OpenSSL and a +/// read buffer +pub struct SslConnection { + stream: SslStream, + buffer: BytesMut, +} + +impl SslConnection { + /// Create a new connection, creating a connection to `host` + pub async fn new(domain: &str, host: &str, sslcert: &str) -> TResult { + let mut connector = SslConnector::builder(SslMethod::tls())?; + connector.set_ca_file(sslcert)?; + let ssl = connector.build().configure()?.into_ssl(&domain)?; + let stream = TcpStream::connect(host).await?; + let mut stream = SslStream::new(ssl, stream)?; + Pin::new(&mut stream).connect().await.unwrap(); + println!("Connected to tps://{}", host); + Ok(SslConnection { + stream, + buffer: BytesMut::with_capacity(BUF_CAP), + }) + } + /// This function will write a query to the stream and read the response from the + /// server. It will then determine if the returned response is complete or incomplete + /// or invalid. + /// + /// - If it is complete, then the return is parsed into a `Display`able form + /// and written to the output stream. If any parsing errors occur, they're also handled + /// by this function (usually, "Invalid Response" is written to the terminal). + /// - If the packet is incomplete, it will wait to read the entire response from the stream + /// - If the packet is corrupted, it will output "Invalid Response" + pub async fn run_query(&mut self, query: String) { + let query = terrapipe::proc_query(query); + println!("Processed query and now connected!"); + match self.stream.write_all(&query).await { + Ok(_) => (), + Err(e) => { + eprintln!("ERROR: Couldn't write data to socket with '{}'", e); + return; + } + }; + loop { + match self.stream.read_buf(&mut self.buffer).await { + Ok(_) => (), + Err(e) => { + eprintln!("ERROR: {}", e); + return; + } + } + println!( + "The server returned: {}", + String::from_utf8_lossy(&self.buffer) + ); match self.try_response().await { ClientResult::Empty(f) => { self.buffer.advance(f); diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index 5e9c0f4a..a68fcfc7 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -197,9 +197,6 @@ impl SslOpts { pub const fn new(key: String, chain: String, port: u16) -> Self { SslOpts { key, chain, port } } - pub fn get_host_port_tuple(&self, host: String) -> impl ToSocketAddrs { - ((host), self.port) - } } #[derive(Debug, PartialEq)] diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index e3e2bf57..f75736a8 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -86,6 +86,7 @@ impl SslListener { }) } async fn accept(&mut self) -> TResult> { + println!("Received connection"); let mut backoff = 1; loop { match self.listener.accept().await { @@ -151,6 +152,7 @@ impl SslConnectionHandler { }; match try_df { Ok(QueryResult::Q(s)) => { + println!("Query: {:?}", s); self.db .execute_query(s, &mut Con::init_secure(&mut self.con)) .await? diff --git a/server/src/resp/mod.rs b/server/src/resp/mod.rs index ff6b043b..005d398b 100644 --- a/server/src/resp/mod.rs +++ b/server/src/resp/mod.rs @@ -21,8 +21,6 @@ //! Utilities for generating responses, which are only used by the `server` //! -use crate::protocol::tls::SslConnection; -use crate::protocol::Connection; use bytes::Bytes; use libtdb::terrapipe::RespCodes; use std::error::Error; From d84234724dbfc7f040d342d81d776567f0c52d7d Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Wed, 13 Jan 2021 13:08:11 +0530 Subject: [PATCH 12/17] Update to upstream changes and polish SSL API Also several debug messages were added for debugging SSL connections. Signed-off-by: Sayan Nandan --- Cargo.lock | 160 ++++++++++++++++++----------------- cli/Cargo.toml | 2 +- cli/src/argparse.rs | 6 +- cli/src/protocol/mod.rs | 19 ++--- server/Cargo.toml | 21 ++--- server/src/dbnet.rs | 2 +- server/src/protocol/tls.rs | 46 +++++----- server/src/resp/mod.rs | 4 +- server/src/tests/kvengine.rs | 4 +- tdb-macros/Cargo.toml | 6 +- tdb-macros/src/lib.rs | 8 +- 11 files changed, 144 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aaedab6a..bfc328ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" [[package]] name = "bytes" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cc" @@ -277,6 +277,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + [[package]] name = "hermit-abi" version = "0.1.15" @@ -357,9 +368,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" dependencies = [ "cfg-if 0.1.10", ] @@ -567,9 +578,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" dependencies = [ "proc-macro2", ] @@ -580,11 +591,23 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.15", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24fcd450d3fa2b592732565aa4f17a27a61c65ece4726353e000939b0edee34" +dependencies = [ "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.3.0", + "rand_core 0.6.1", + "rand_hc 0.3.0", ] [[package]] @@ -594,7 +617,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.1", ] [[package]] @@ -603,7 +636,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.15", +] + +[[package]] +name = "rand_core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +dependencies = [ + "getrandom 0.2.1", ] [[package]] @@ -612,7 +654,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.1", ] [[package]] @@ -653,18 +704,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "9bdd36f49e35b61d49efd8aa7fc068fd295961fd2286d0b2ee9a4c7a14e99cc3" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "552954ce79a059ddd5fd68c271592374bd15cab2274970380c000118aeffe1cd" dependencies = [ "proc-macro2", "quote", @@ -724,9 +775,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.54" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2", "quote", @@ -738,10 +789,11 @@ name = "tdb" version = "0.5.0" dependencies = [ "bincode", - "bytes 0.6.0", + "bytes 1.0.1", "chrono", "clap", "env_logger", + "futures", "jemallocator", "lazy_static", "libtdb", @@ -753,8 +805,8 @@ dependencies = [ "serde", "serde_derive", "tdb_macros", - "tokio 0.3.6", - "tokio-openssl 0.5.0", + "tokio", + "tokio-openssl", "toml", ] @@ -765,7 +817,7 @@ dependencies = [ "clap", "devtimer", "libtdb", - "rand", + "rand 0.7.3", "serde", "serde_json", ] @@ -776,7 +828,7 @@ version = "0.2.2" dependencies = [ "proc-macro2", "quote", - "rand", + "rand 0.8.1", "syn", ] @@ -818,28 +870,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "tokio" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "720ba21c25078711bf456d607987d95bce90f7c3bea5abe1db587862e7a1e87c" -dependencies = [ - "autocfg", - "bytes 0.6.0", - "futures-core", - "libc", - "memchr", - "mio", - "num_cpus", - "once_cell", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "tokio-macros 0.3.1", - "winapi", -] - [[package]] name = "tokio" version = "1.0.1" @@ -847,7 +877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd" dependencies = [ "autocfg", - "bytes 1.0.0", + "bytes 1.0.1", "libc", "memchr", "mio", @@ -856,21 +886,10 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "tokio-macros 1.0.0", + "tokio-macros", "winapi", ] -[[package]] -name = "tokio-macros" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d30fdbb5dc2d8f91049691aa1a9d4d4ae422a21c334ce8936e5886d30c5c45" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-macros" version = "1.0.0" @@ -882,17 +901,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-openssl" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01e5cc2d3a154fa310982d0affaec8140d6476805422265b2f648eb813f937f" -dependencies = [ - "openssl", - "openssl-sys", - "tokio 0.3.6", -] - [[package]] name = "tokio-openssl" version = "0.6.1" @@ -902,14 +910,14 @@ dependencies = [ "futures", "openssl", "pin-project", - "tokio 1.0.1", + "tokio", ] [[package]] name = "toml" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ "serde", ] @@ -918,14 +926,14 @@ dependencies = [ name = "tsh" version = "0.5.0" dependencies = [ - "bytes 1.0.0", + "bytes 1.0.1", "clap", "lazy_static", "libtdb", "openssl", "regex", - "tokio 1.0.1", - "tokio-openssl 0.6.1", + "tokio", + "tokio-openssl", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 3a15caec..cde954d5 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] libtdb = {path = "../libtdb"} tokio = {version = "1.0.1", features = ["full"]} -bytes = "1.0.0" +bytes = "1.0.1" regex = "1.4.3" lazy_static = "1.4.0" clap = {version = "2.33.3", features=["yaml"]} diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs index 568a3511..f981789f 100644 --- a/cli/src/argparse.rs +++ b/cli/src/argparse.rs @@ -40,7 +40,6 @@ pub async fn start_repl() { Some(h) => h.to_owned(), None => ADDR.to_owned(), }; - let domain = host.clone(); host.push(':'); match matches.value_of("port") { Some(p) => match p.parse::() { @@ -52,8 +51,9 @@ pub async fn start_repl() { }, None => host.push_str("2003"), } - let mut con = if let Some(sslcert) = matches.value_of("cert") { - let con = match SslConnection::new(&domain, &host, sslcert).await { + let ssl = matches.value_of("cert"); + let mut con = if let Some(sslcert) = ssl { + let con = match SslConnection::new(&host, sslcert).await { Ok(c) => c, Err(e) => { eprintln!("ERROR: {}", e); diff --git a/cli/src/protocol/mod.rs b/cli/src/protocol/mod.rs index 280f0423..85b643b9 100644 --- a/cli/src/protocol/mod.rs +++ b/cli/src/protocol/mod.rs @@ -26,7 +26,8 @@ use lazy_static::lazy_static; use libtdb::terrapipe; use libtdb::TResult; use libtdb::BUF_CAP; -use openssl::ssl::SslConnector; +use openssl::ssl::Ssl; +use openssl::ssl::SslContext; use openssl::ssl::SslMethod; use regex::Regex; use std::pin::Pin; @@ -94,10 +95,6 @@ impl Connection { return; } } - println!( - "The server returned: {}", - String::from_utf8_lossy(&self.buffer) - ); match self.try_response().await { ClientResult::Empty(f) => { self.buffer.advance(f); @@ -144,10 +141,10 @@ pub struct SslConnection { impl SslConnection { /// Create a new connection, creating a connection to `host` - pub async fn new(domain: &str, host: &str, sslcert: &str) -> TResult { - let mut connector = SslConnector::builder(SslMethod::tls())?; - connector.set_ca_file(sslcert)?; - let ssl = connector.build().configure()?.into_ssl(&domain)?; + pub async fn new(host: &str, sslcert: &str) -> TResult { + let mut ctx = SslContext::builder(SslMethod::tls_client())?; + ctx.set_ca_file(sslcert)?; + let ssl = Ssl::new(&ctx.build())?; let stream = TcpStream::connect(host).await?; let mut stream = SslStream::new(ssl, stream)?; Pin::new(&mut stream).connect().await.unwrap(); @@ -184,10 +181,6 @@ impl SslConnection { return; } } - println!( - "The server returned: {}", - String::from_utf8_lossy(&self.buffer) - ); match self.try_response().await { ClientResult::Empty(f) => { self.buffer.advance(f); diff --git a/server/Cargo.toml b/server/Cargo.toml index 7efd6911..e6b69ffd 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,26 +7,27 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "0.3.6", features = ["full"] } -bytes = "0.6.0" +tokio = { version = "1.0.1", features = ["full"] } +bytes = "1.0.1" libtdb = {path ="../libtdb"} bincode = "1.3.1" parking_lot = "0.11.1" lazy_static = "1.4.0" -serde_derive = "1.0.117" -serde = {version = "1.0.118", features= ["derive"]} -toml = "0.5.7" +serde_derive = "1.0.119" +futures = "0.3.9" +serde = {version = "1.0.119", features= ["derive"]} +toml = "0.5.8" clap = {version = "2.33.3", features=["yaml"]} env_logger = "0.8.2" -log = "0.4.11" +log = "0.4.13" chrono = "0.4.19" -regex = "1.4.2" +regex = "1.4.3" tdb_macros = {path="../tdb-macros"} -tokio-openssl = "0.5.0" +tokio-openssl = "0.6.1" openssl = { version = "0.10", features = ["vendored"] } -openssl-sys = "0.9.59" +openssl-sys = "0.9.60" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.3.2" [dev-dependencies] -tokio = { version = "0.3.6", features = ["test-util"] } +tokio = { version = "1.0.1", features = ["test-util"] } diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index c049ac8f..cd3c3110 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -144,7 +144,7 @@ impl Listener { loop { // Take the permit first, but we won't use it right now // that's why we will forget it - self.climit.acquire().await.forget(); + self.climit.acquire().await.unwrap().forget(); let stream = self.accept().await?; let mut chandle = CHandler { db: self.db.clone(), diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index f75736a8..95dd8066 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -30,15 +30,15 @@ use crate::resp::Writable; use crate::CoreDB; use bytes::Buf; use bytes::BytesMut; +use futures::future; use libtdb::TResult; use libtdb::BUF_CAP; -use openssl::ssl::{SslAcceptor, SslFiletype, SslMethod}; +use openssl::ssl::{Ssl, SslAcceptor, SslFiletype, SslMethod}; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::Arc; -use tokio::io::BufWriter; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; -use tokio::net::TcpStream; +use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::Semaphore; use tokio::sync::{broadcast, mpsc}; use tokio::time::{self, Duration}; @@ -57,7 +57,7 @@ pub struct SslListener { // We send a clone of `terminate_tx` to each `CHandler` pub terminate_tx: mpsc::Sender<()>, pub terminate_rx: mpsc::Receiver<()>, - acceptor: Arc, + acceptor: SslAcceptor, } impl SslListener { @@ -71,10 +71,11 @@ impl SslListener { terminate_tx: mpsc::Sender<()>, terminate_rx: mpsc::Receiver<()>, ) -> TResult { + log::debug!("New SSL/TLS connection registered"); let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls())?; acceptor.set_private_key_file(key_file, SslFiletype::PEM)?; acceptor.set_certificate_chain_file(chain_file)?; - let acceptor = Arc::new(acceptor.build()); + let acceptor = acceptor.build(); Ok(SslListener { db, listener, @@ -86,19 +87,23 @@ impl SslListener { }) } async fn accept(&mut self) -> TResult> { - println!("Received connection"); + log::debug!("Trying to accept a SSL connection"); let mut backoff = 1; loop { match self.listener.accept().await { // We don't need the bindaddr // We get the encrypted stream which we need to decrypt // by using the acceptor - Ok((encrypted_stream, _)) => { - let decrypted_stream = - tokio_openssl::accept(&self.acceptor, encrypted_stream).await?; - return Ok(decrypted_stream); + Ok((stream, _)) => { + log::debug!("Accepted an SSL/TLS connection"); + let ssl = Ssl::new(self.acceptor.context())?; + let mut stream = SslStream::new(ssl, stream)?; + Pin::new(&mut stream).accept().await?; + log::debug!("Connected to secure socket over TCP"); + return Ok(stream); } Err(e) => { + log::debug!("Failed to establish a secure connection"); if backoff > 64 { // Too many retries, goodbye user return Err(e.into()); @@ -112,10 +117,11 @@ impl SslListener { } } pub async fn run(&mut self) -> TResult<()> { + log::debug!("Started secure server"); loop { // Take the permit first, but we won't use it right now // that's why we will forget it - self.climit.acquire().await.forget(); + self.climit.acquire().await.unwrap().forget(); let stream = self.accept().await?; let mut sslhandle = SslConnectionHandler { db: self.db.clone(), @@ -125,6 +131,7 @@ impl SslListener { _term_sig_tx: self.terminate_tx.clone(), }; tokio::spawn(async move { + log::debug!("Spawned listener task"); if let Err(e) = sslhandle.run().await { eprintln!("Error: {}", e); } @@ -143,6 +150,7 @@ pub struct SslConnectionHandler { impl SslConnectionHandler { pub async fn run(&mut self) -> TResult<()> { + log::debug!("SslConnectionHanler initialized to handle a remote client"); while !self.terminator.is_termination_signal() { let try_df = tokio::select! { tdf = self.con.read_query() => tdf, @@ -174,19 +182,19 @@ impl Drop for SslConnectionHandler { } pub struct SslConnection { - stream: BufWriter>, + stream: SslStream, buffer: BytesMut, } impl SslConnection { pub fn new(stream: SslStream) -> Self { SslConnection { - stream: BufWriter::new(stream), + stream: stream, buffer: BytesMut::with_capacity(BUF_CAP), } } async fn read_again(&mut self) -> Result<(), String> { - match self.stream.read_buf(&mut self.buffer).await { + match self.stream.get_mut().read_buf(&mut self.buffer).await { Ok(0) => { // If 0 bytes were received, then the remote end closed // the connection @@ -209,7 +217,7 @@ impl SslConnection { } } fn get_peer(&self) -> IoResult { - self.stream.get_ref().get_ref().peer_addr() + self.stream.get_ref().peer_addr() } /// Try to parse a query from the buffered data fn try_query(&mut self) -> Result { @@ -244,14 +252,14 @@ impl SslConnection { Ok(()) } pub async fn flush_stream(&mut self) -> TResult<()> { - self.stream.flush().await?; + self.stream.get_mut().flush().await?; Ok(()) } /// Wraps around the `write_response` used to differentiate between a /// success response and an error response pub async fn close_conn_with_error(&mut self, resp: Vec) -> TResult<()> { self.write_response(resp).await?; - self.stream.flush().await?; + self.stream.get_mut().flush().await?; Ok(()) } } diff --git a/server/src/resp/mod.rs b/server/src/resp/mod.rs index 005d398b..b7ef9fd9 100644 --- a/server/src/resp/mod.rs +++ b/server/src/resp/mod.rs @@ -63,12 +63,12 @@ impl IsConnection for BufWriter { } } -impl IsConnection for BufWriter> { +impl IsConnection for SslStream { fn write_lowlevel<'s>( &'s mut self, bytes: &'s [u8], ) -> Pin> + Send + Sync + 's>> { - Box::pin(self.write(bytes)) + Box::pin(self.get_mut().write(bytes)) } } diff --git a/server/src/tests/kvengine.rs b/server/src/tests/kvengine.rs index 037bcb6c..163f7f57 100644 --- a/server/src/tests/kvengine.rs +++ b/server/src/tests/kvengine.rs @@ -24,7 +24,7 @@ mod __private { use crate::protocol::responses::fresp; use libtdb::terrapipe; use tokio::io::AsyncReadExt; - use tokio::prelude::*; + use tokio::io::AsyncWriteExt; /// Test a HEYA query: The server should return HEY! async fn test_heya() { let heya = terrapipe::proc_query("HEYA"); @@ -82,7 +82,7 @@ mod __private { ) where T: AsRef, { - use tokio::prelude::*; + use tokio::io::AsyncWriteExt; let mut query = String::from("MSET "); query.push_str(values_split_with_whitespace.as_ref()); let count_bytes_len = homwany.to_string().as_bytes().len(); diff --git a/tdb-macros/Cargo.toml b/tdb-macros/Cargo.toml index a44f4864..c0a39227 100644 --- a/tdb-macros/Cargo.toml +++ b/tdb-macros/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" proc-macro = true [dependencies] -syn = {version = "1.0.54", features = ["full"]} -quote = "1.0.7" -rand = "0.7.3" +syn = {version = "1.0.58", features = ["full"]} +quote = "1.0.8" +rand = "0.8.1" proc-macro2 = "1.0.24" diff --git a/tdb-macros/src/lib.rs b/tdb-macros/src/lib.rs index 14eb3728..58e48e05 100644 --- a/tdb-macros/src/lib.rs +++ b/tdb-macros/src/lib.rs @@ -67,7 +67,7 @@ fn parse_dbtest(mut input: syn::ItemFn, rand: u16) -> Result TokenStream { over by Hyper-V, which is why we'll prevent attempts to bind to them, if the OS is Windows. */ - let mut rand: u16 = rng.gen_range(1025, 65535); + let mut rand: u16 = rng.gen_range(1025..=65535); #[cfg(not(target_os = "windows"))] { while in_set.contains(&rand) { - rand = rng.gen_range(1025, 65535); + rand = rng.gen_range(1025..=65535); } } #[cfg(target_os = "windows")] @@ -195,7 +195,7 @@ fn parse_test_module(args: TokenStream, item: TokenStream) -> TokenStream { in_set.insert(5357); in_set.insert(7680); while in_set.contains(&rand) || (rand >= 49670 && rand <= 50293) { - rand = rng.gen_range(1025, 65535); + rand = rng.gen_range(1025..=65535); } } in_set.insert(rand); From c74259a488752b47233ceb74d46d4304a98778c5 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Wed, 13 Jan 2021 13:43:53 +0530 Subject: [PATCH 13/17] Listen to futures in parallel with `join!` Signed-off-by: Sayan Nandan --- server/src/dbnet.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index cd3c3110..d9df9b8c 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -354,8 +354,15 @@ impl MultiListener { MultiListener::SecureOnly(secure_listener) => secure_listener.run().await, MultiListener::InsecureOnly(insecure_listener) => insecure_listener.run().await, MultiListener::Multi(insecure_listener, secure_listener) => { - insecure_listener.run().await?; - secure_listener.run().await?; + let insec = insecure_listener.run(); + let sec = secure_listener.run(); + let (e1, e2) = futures::join!(insec, sec); + if let Err(e) = e1 { + log::error!("Insecure listener failed with: {}", e); + } + if let Err(e) = e2 { + log::error!("Secure listener failed with: {}", e); + } Ok(()) } } From 8c067d20a57faa77d81dfa218cb684d0ab0e7bf5 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Wed, 13 Jan 2021 17:36:58 +0530 Subject: [PATCH 14/17] Fix writing to TCP socket instead of SSL socket We were doing an extremely erroneous thing: writing to the TCP socket instead of the SSL socket. This caused OpenSSL to report problems on the client and server sides, telling us that there was a problem with the SSL connection. This commit revises the `write_lowlevel` trait impl for `SslStream` to write to the SSL socket. Also, the 'wrong' flushing of data for similar reasons has been fixed Signed-off-by: Sayan Nandan --- cli/src/protocol/mod.rs | 3 +-- server/src/dbnet.rs | 4 ++-- server/src/protocol/tls.rs | 16 +++++++++------- server/src/resp/mod.rs | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cli/src/protocol/mod.rs b/cli/src/protocol/mod.rs index 85b643b9..4e5cebb5 100644 --- a/cli/src/protocol/mod.rs +++ b/cli/src/protocol/mod.rs @@ -165,7 +165,6 @@ impl SslConnection { /// - If the packet is corrupted, it will output "Invalid Response" pub async fn run_query(&mut self, query: String) { let query = terrapipe::proc_query(query); - println!("Processed query and now connected!"); match self.stream.write_all(&query).await { Ok(_) => (), Err(e) => { @@ -177,7 +176,7 @@ impl SslConnection { match self.stream.read_buf(&mut self.buffer).await { Ok(_) => (), Err(e) => { - eprintln!("ERROR: {}", e); + eprintln!("ERROR: Couldn't read data from socket with '{}'", e); return; } } diff --git a/server/src/dbnet.rs b/server/src/dbnet.rs index d9df9b8c..6829c5b5 100644 --- a/server/src/dbnet.rs +++ b/server/src/dbnet.rs @@ -155,7 +155,7 @@ impl Listener { }; tokio::spawn(async move { if let Err(e) = chandle.run().await { - eprintln!("Error: {}", e); + log::error!("Error: {}", e); } }); } @@ -460,7 +460,7 @@ pub async fn run( let db = match CoreDB::new(bgsave_cfg, snapshot_cfg, restore_filepath) { Ok(d) => d, Err(e) => { - eprintln!("ERROR: {}", e); + log::error!("ERROR: {}", e); process::exit(0x100); } }; diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 95dd8066..bcdb4e4a 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -30,14 +30,13 @@ use crate::resp::Writable; use crate::CoreDB; use bytes::Buf; use bytes::BytesMut; -use futures::future; use libtdb::TResult; use libtdb::BUF_CAP; use openssl::ssl::{Ssl, SslAcceptor, SslFiletype, SslMethod}; use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; -use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::Semaphore; use tokio::sync::{broadcast, mpsc}; @@ -133,7 +132,7 @@ impl SslListener { tokio::spawn(async move { log::debug!("Spawned listener task"); if let Err(e) = sslhandle.run().await { - eprintln!("Error: {}", e); + log::error!("Error: {}", e); } }); } @@ -165,7 +164,10 @@ impl SslConnectionHandler { .execute_query(s, &mut Con::init_secure(&mut self.con)) .await? } - Ok(QueryResult::E(r)) => self.con.close_conn_with_error(r).await?, + Ok(QueryResult::E(r)) => { + log::debug!("Failed to read query!"); + self.con.close_conn_with_error(r).await? + } Ok(QueryResult::Empty) => return Ok(()), Err(e) => return Err(e.into()), } @@ -194,7 +196,7 @@ impl SslConnection { } } async fn read_again(&mut self) -> Result<(), String> { - match self.stream.get_mut().read_buf(&mut self.buffer).await { + match self.stream.read_buf(&mut self.buffer).await { Ok(0) => { // If 0 bytes were received, then the remote end closed // the connection @@ -252,14 +254,14 @@ impl SslConnection { Ok(()) } pub async fn flush_stream(&mut self) -> TResult<()> { - self.stream.get_mut().flush().await?; + self.stream.flush().await?; Ok(()) } /// Wraps around the `write_response` used to differentiate between a /// success response and an error response pub async fn close_conn_with_error(&mut self, resp: Vec) -> TResult<()> { self.write_response(resp).await?; - self.stream.get_mut().flush().await?; + self.stream.flush().await?; Ok(()) } } diff --git a/server/src/resp/mod.rs b/server/src/resp/mod.rs index b7ef9fd9..02231c28 100644 --- a/server/src/resp/mod.rs +++ b/server/src/resp/mod.rs @@ -68,7 +68,7 @@ impl IsConnection for SslStream { &'s mut self, bytes: &'s [u8], ) -> Pin> + Send + Sync + 's>> { - Box::pin(self.get_mut().write(bytes)) + Box::pin(self.write(bytes)) } } From e92a6eb8c1197670a952645771c109cbf3aa9e39 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Fri, 15 Jan 2021 16:42:33 +0530 Subject: [PATCH 15/17] Use `read_again()` to read from the `SslStream` Signed-off-by: Sayan Nandan --- Cargo.lock | 169 ++++++++++++++++--------------------- cli/Cargo.toml | 2 +- cli/src/protocol/mod.rs | 39 +++++++-- libtdb/Cargo.toml | 4 +- server/Cargo.toml | 6 +- server/src/protocol/tls.rs | 1 - 6 files changed, 111 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfc328ff..f21092a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "aho-corasick" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" dependencies = [ "memchr", ] @@ -18,12 +18,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "arc-swap" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" - [[package]] name = "atty" version = "0.2.14" @@ -59,15 +53,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - -[[package]] -name = "bytes" -version = "0.6.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" @@ -77,9 +65,9 @@ checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cc" -version = "1.0.59" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -122,20 +110,11 @@ dependencies = [ "yaml-rust", ] -[[package]] -name = "cloudabi" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] - [[package]] name = "devtimer" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6035b7b9244bf9637cd7ef80b5e1c54404bef92cccd34738c85c45f04ae8b244" +checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1" [[package]] name = "env_logger" @@ -173,9 +152,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70be434c505aee38639abccb918163b63158a4b4bb791b45b7023044bdc3c9c" +checksum = "90fa4cc29d25b0687b8570b0da86eac698dcb525110ad8b938fe6712baa711ec" dependencies = [ "futures-channel", "futures-core", @@ -188,9 +167,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f01c61843314e95f96cc9245702248733a3a3d744e43e2e755e3c7af8348a0a9" +checksum = "31ebc390c6913de330e418add60e1a7e5af4cb5ec600d19111b339cafcdcc027" dependencies = [ "futures-core", "futures-sink", @@ -198,15 +177,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8d3b0917ff63a2a96173133c02818fac4a746b0a57569d3baca9ec0e945e08" +checksum = "089bd0baf024d3216916546338fffe4fc8dfffdd901e33c278abb091e0d52111" [[package]] name = "futures-executor" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee9ca2f7eb4475772cf39dd1cd06208dce2670ad38f4d9c7262b3e15f127068" +checksum = "d0cb59f15119671c94cd9cc543dc9a50b8d5edc468b4ff5f0bb8567f66c6b48a" dependencies = [ "futures-core", "futures-task", @@ -215,15 +194,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37c1a51b037b80922864b8eed90692c5cd8abd4c71ce49b77146caa47f3253b" +checksum = "3868967e4e5ab86614e2176c99949eeef6cbcacaee737765f6ae693988273997" [[package]] name = "futures-macro" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f8719ca0e1f3c5e34f3efe4570ef2c0610ca6da85ae7990d472e9cbfba13664" +checksum = "95778720c3ee3c179cd0d8fd5a0f9b40aa7d745c080f86a8f8bed33c4fd89758" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -233,24 +212,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6adabac1290109cfa089f79192fb6244ad2c3f1cc2281f3e1dd987592b71feb" +checksum = "d4e0f6be0ec0357772fd58fb751958dd600bd0b3edfd429e77793e4282831360" [[package]] name = "futures-task" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92a0843a2ff66823a8f7c77bffe9a09be2b64e533562c412d63075643ec0038" +checksum = "868090f28a925db6cb7462938c51d807546e298fb314088239f0e52fb4338b96" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9" +checksum = "cad5e82786df758d407932aded1235e24d8e2eb438b6adafd37930c2462fb5d1" dependencies = [ "futures-channel", "futures-core", @@ -268,11 +247,11 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -290,9 +269,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" dependencies = [ "libc", ] @@ -305,15 +284,18 @@ checksum = "3c1ad908cc71012b7bea4d0c53ba96a8cba9962f048fa68d143376143d863b7a" [[package]] name = "instant" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jemalloc-sys" @@ -344,24 +326,24 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.72" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" [[package]] name = "libtdb" version = "0.5.0" dependencies = [ - "bytes 0.6.0", + "bytes", "lazy_static", "termcolor", ] [[package]] name = "lock_api" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" dependencies = [ "scopeguard", ] @@ -377,15 +359,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "mio" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b" +checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" dependencies = [ "libc", "log", @@ -415,9 +397,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", "num-traits", @@ -425,9 +407,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg", ] @@ -498,12 +480,11 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" dependencies = [ - "cfg-if 0.1.10", - "cloudabi", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -533,9 +514,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" [[package]] name = "pin-utils" @@ -551,9 +532,9 @@ checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "ppv-lite86" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "proc-macro-hack" @@ -591,7 +572,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -636,7 +617,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.15", + "getrandom 0.1.16", ] [[package]] @@ -724,9 +705,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" dependencies = [ "itoa", "ryu", @@ -735,11 +716,10 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" dependencies = [ - "arc-swap", "libc", ] @@ -751,19 +731,18 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.3.16" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "redox_syscall", "winapi", ] @@ -789,7 +768,7 @@ name = "tdb" version = "0.5.0" dependencies = [ "bincode", - "bytes 1.0.1", + "bytes", "chrono", "clap", "env_logger", @@ -872,12 +851,12 @@ dependencies = [ [[package]] name = "tokio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd" +checksum = "0ca04cec6ff2474c638057b65798f60ac183e5e79d3448bb7163d36a39cff6ec" dependencies = [ "autocfg", - "bytes 1.0.1", + "bytes", "libc", "memchr", "mio", @@ -926,7 +905,7 @@ dependencies = [ name = "tsh" version = "0.5.0" dependencies = [ - "bytes 1.0.1", + "bytes", "clap", "lazy_static", "libtdb", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index cde954d5..f29a1bf7 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] libtdb = {path = "../libtdb"} -tokio = {version = "1.0.1", features = ["full"]} +tokio = {version = "1.0.2", features = ["full"]} bytes = "1.0.1" regex = "1.4.3" lazy_static = "1.4.0" diff --git a/cli/src/protocol/mod.rs b/cli/src/protocol/mod.rs index 4e5cebb5..5675515e 100644 --- a/cli/src/protocol/mod.rs +++ b/cli/src/protocol/mod.rs @@ -30,6 +30,8 @@ use openssl::ssl::Ssl; use openssl::ssl::SslContext; use openssl::ssl::SslMethod; use regex::Regex; +use std::io::Result as IoResult; +use std::net::SocketAddr; use std::pin::Pin; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -173,12 +175,9 @@ impl SslConnection { } }; loop { - match self.stream.read_buf(&mut self.buffer).await { - Ok(_) => (), - Err(e) => { - eprintln!("ERROR: Couldn't read data from socket with '{}'", e); - return; - } + if let Err(e) = self.read_again().await { + eprintln!("ERROR: Reading from stream failed with: '{}'", e); + return; } match self.try_response().await { ClientResult::Empty(f) => { @@ -199,8 +198,8 @@ impl SslConnection { } return; } - ClientResult::InvalidResponse(_) => { - self.buffer.clear(); + ClientResult::InvalidResponse(r) => { + self.buffer.advance(r); eprintln!("{}", ClientResult::InvalidResponse(0)); return; } @@ -215,4 +214,28 @@ impl SslConnection { } deserializer::parse(&self.buffer) } + async fn read_again(&mut self) -> Result<(), String> { + match self.stream.read_buf(&mut self.buffer).await { + Ok(0) => { + if self.buffer.is_empty() { + return Ok(()); + } else { + return Err(format!( + "Connection reset while reading from {}", + if let Ok(p) = self.get_peer() { + p.to_string() + } else { + "peer".to_owned() + } + ) + .into()); + } + } + Ok(_) => Ok(()), + Err(e) => return Err(format!("{}", e)), + } + } + fn get_peer(&self) -> IoResult { + self.stream.get_ref().peer_addr() + } } diff --git a/libtdb/Cargo.toml b/libtdb/Cargo.toml index 35a8cfd4..7a85dfcd 100644 --- a/libtdb/Cargo.toml +++ b/libtdb/Cargo.toml @@ -8,5 +8,5 @@ edition = "2018" [dependencies] lazy_static = "1.4.0" -bytes = "0.6.0" -termcolor = "1.1.1" \ No newline at end of file +bytes = "1.0.1" +termcolor = "1.1.2" \ No newline at end of file diff --git a/server/Cargo.toml b/server/Cargo.toml index e6b69ffd..ce3a42ee 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,14 +7,14 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.0.1", features = ["full"] } +tokio = { version = "1.0.2", features = ["full"] } bytes = "1.0.1" libtdb = {path ="../libtdb"} bincode = "1.3.1" parking_lot = "0.11.1" lazy_static = "1.4.0" serde_derive = "1.0.119" -futures = "0.3.9" +futures = "0.3.11" serde = {version = "1.0.119", features= ["derive"]} toml = "0.5.8" clap = {version = "2.33.3", features=["yaml"]} @@ -30,4 +30,4 @@ openssl-sys = "0.9.60" jemallocator = "0.3.2" [dev-dependencies] -tokio = { version = "1.0.1", features = ["test-util"] } +tokio = { version = "1.0.2", features = ["test-util"] } diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index bcdb4e4a..126e386d 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -159,7 +159,6 @@ impl SslConnectionHandler { }; match try_df { Ok(QueryResult::Q(s)) => { - println!("Query: {:?}", s); self.db .execute_query(s, &mut Con::init_secure(&mut self.con)) .await? From d3387b68d075ff6270cd08f30ab29a748317413b Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Fri, 15 Jan 2021 17:21:56 +0530 Subject: [PATCH 16/17] Fix erratic packet delivery times with `BufWriter` The data was being delievered in different batches, which caused problems. This commit replaces the current stream writer with a buffered writer ensuring good delivery. Signed-off-by: Sayan Nandan --- server/src/protocol/tls.rs | 7 ++++--- server/src/resp/mod.rs | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/src/protocol/tls.rs b/server/src/protocol/tls.rs index 126e386d..d9aa3242 100644 --- a/server/src/protocol/tls.rs +++ b/server/src/protocol/tls.rs @@ -36,6 +36,7 @@ use openssl::ssl::{Ssl, SslAcceptor, SslFiletype, SslMethod}; use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; +use tokio::io::BufWriter; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::Semaphore; @@ -183,14 +184,14 @@ impl Drop for SslConnectionHandler { } pub struct SslConnection { - stream: SslStream, + stream: BufWriter>, buffer: BytesMut, } impl SslConnection { pub fn new(stream: SslStream) -> Self { SslConnection { - stream: stream, + stream: BufWriter::new(stream), buffer: BytesMut::with_capacity(BUF_CAP), } } @@ -218,7 +219,7 @@ impl SslConnection { } } fn get_peer(&self) -> IoResult { - self.stream.get_ref().peer_addr() + self.stream.get_ref().get_ref().peer_addr() } /// Try to parse a query from the buffered data fn try_query(&mut self) -> Result { diff --git a/server/src/resp/mod.rs b/server/src/resp/mod.rs index 02231c28..a4c652ef 100644 --- a/server/src/resp/mod.rs +++ b/server/src/resp/mod.rs @@ -72,6 +72,15 @@ impl IsConnection for SslStream { } } +impl IsConnection for BufWriter> { + fn write_lowlevel<'s>( + &'s mut self, + bytes: &'s [u8], + ) -> Pin> + Send + Sync + 's>> { + Box::pin(self.write(bytes)) + } +} + /// A `BytesWrapper` object wraps around a `Bytes` object that might have been pulled /// from `CoreDB`. /// From 4722e01d2355d7863eee67faa8f96f6e58bf0201 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Sat, 16 Jan 2021 09:10:02 +0530 Subject: [PATCH 17/17] Add `ran_string()` to `tdb-bench` Also, dependencies were upgraded across all crates and the version for `tdb-macros` was streamlined to 0.5.0 like the other crates. Signed-off-by: Sayan Nandan --- Cargo.lock | 84 +++++------------------------ cli/src/argparse.rs | 2 +- cli/src/cli.yml | 2 +- examples/config-files/template.toml | 2 +- server/src/cli.yml | 2 +- server/src/config/mod.rs | 2 +- server/src/main.rs | 2 +- tdb-bench/Cargo.toml | 8 +-- tdb-bench/src/cli.yml | 2 +- tdb-bench/src/main.rs | 37 ++++++------- tdb-macros/Cargo.toml | 4 +- 11 files changed, 42 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f21092a5..c6fb1c9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,17 +245,6 @@ dependencies = [ "slab", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.1" @@ -264,7 +253,7 @@ checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -568,37 +557,14 @@ dependencies = [ [[package]] name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24fcd450d3fa2b592732565aa4f17a27a61c65ece4726353e000939b0edee34" +checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" dependencies = [ "libc", - "rand_chacha 0.3.0", - "rand_core 0.6.1", - "rand_hc 0.3.0", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", + "rand_hc", ] [[package]] @@ -608,16 +574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.1", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -626,16 +583,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" dependencies = [ - "getrandom 0.2.1", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "getrandom", ] [[package]] @@ -644,7 +592,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core 0.6.1", + "rand_core", ] [[package]] @@ -796,18 +744,18 @@ dependencies = [ "clap", "devtimer", "libtdb", - "rand 0.7.3", + "rand", "serde", "serde_json", ] [[package]] name = "tdb_macros" -version = "0.2.2" +version = "0.5.0" dependencies = [ "proc-macro2", "quote", - "rand 0.8.1", + "rand", "syn", ] @@ -845,7 +793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi", "winapi", ] @@ -939,12 +887,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs index f981789f..c5513a1b 100644 --- a/cli/src/argparse.rs +++ b/cli/src/argparse.rs @@ -26,7 +26,7 @@ use libtdb::terrapipe::ADDR; use protocol::{Con, Connection, SslConnection}; use std::io::{self, prelude::*}; use std::process; -const MSG_WELCOME: &'static str = "TerrabaseDB v0.5.0"; +const MSG_WELCOME: &'static str = "TerrabaseDB v0.5.1"; /// This creates a REPL on the command line and also parses command-line arguments /// diff --git a/cli/src/cli.yml b/cli/src/cli.yml index bd6ae5fb..a666c600 100644 --- a/cli/src/cli.yml +++ b/cli/src/cli.yml @@ -20,7 +20,7 @@ # name: TerrabaseDB Shell -version: 0.5.0 +version: 0.5.1 author: Sayan N. about: The TerrabaseDB Shell (tsh) args: diff --git a/examples/config-files/template.toml b/examples/config-files/template.toml index 697aa9db..bee655f1 100644 --- a/examples/config-files/template.toml +++ b/examples/config-files/template.toml @@ -12,7 +12,7 @@ port = 2003 # The port to which you want TDB to bind to # Set `noart` to true if you want to disable terminal artwork noart = false -# This key is *OPTIONAL*, but will be required post 0.5.0 +# This key is *OPTIONAL*, but will be required post 0.5.1 [bgsave] # Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run # after every 2 minutes diff --git a/server/src/cli.yml b/server/src/cli.yml index 2d7be12f..01940e98 100644 --- a/server/src/cli.yml +++ b/server/src/cli.yml @@ -1,5 +1,5 @@ name: TerrabaseDB Server -version: 0.5.0 +version: 0.5.1 author: Sayan N. about: The TerrabaseDB Database server args: diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs index a68fcfc7..09bb5e1a 100644 --- a/server/src/config/mod.rs +++ b/server/src/config/mod.rs @@ -44,7 +44,7 @@ pub struct Config { /// The `server` key server: ConfigKeyServer, /// The `bgsave` key - /* TODO(@ohsayan): As of now, we will keep this optional, but post 0.5.0, + /* TODO(@ohsayan): As of now, we will keep this optional, but post 0.5.1, * we will make it compulsory (so that we don't break semver) * See the link below for more details: * https://github.com/terrabasedb/terrabasedb/issues/21#issuecomment-693217709 diff --git a/server/src/main.rs b/server/src/main.rs index e71ad47b..4ccbfd0a 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -54,7 +54,7 @@ use jemallocator::Jemalloc; static GLOBAL: Jemalloc = Jemalloc; /// The version text -static MSG: &'static str = "TerrabaseDB v0.5.0 | https://github.com/terrabasedb/terrabasedb"; +static MSG: &'static str = "TerrabaseDB v0.5.1 | https://github.com/terrabasedb/terrabasedb"; /// The terminal art for `!noart` configurations static TEXT: &'static str = " _______ _ _____ ____ diff --git a/tdb-bench/Cargo.toml b/tdb-bench/Cargo.toml index 92bd6305..dcbf6198 100644 --- a/tdb-bench/Cargo.toml +++ b/tdb-bench/Cargo.toml @@ -7,9 +7,9 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rand = "0.7.3" -devtimer = "4.0.0" +rand = "0.8.2" +devtimer = "4.0.1" libtdb = {path="../libtdb"} clap = {version = "2.33.3", features=["yaml"]} -serde = {version="1.0.118", features=["derive"]} -serde_json = "1.0.60" +serde = {version="1.0.119", features=["derive"]} +serde_json = "1.0.61" diff --git a/tdb-bench/src/cli.yml b/tdb-bench/src/cli.yml index ae09dc82..2080f7da 100644 --- a/tdb-bench/src/cli.yml +++ b/tdb-bench/src/cli.yml @@ -20,7 +20,7 @@ # name: TerrabaseDB Benchmark Tool -version: 0.5.0 +version: 0.5.1 author: Sayan N. about: | The TerrabaseDB benchmark tool can be used to benchmark TerrabaseDB installations. diff --git a/tdb-bench/src/main.rs b/tdb-bench/src/main.rs index cc196dca..eef8c61b 100644 --- a/tdb-bench/src/main.rs +++ b/tdb-bench/src/main.rs @@ -28,7 +28,7 @@ mod benchtool { use devtimer::DevTime; use libtdb::terrapipe; use rand::distributions::Alphanumeric; - use rand::{thread_rng, Rng}; + use rand::thread_rng; use serde::Serialize; use std::io::prelude::*; use std::net::{self, TcpStream}; @@ -167,7 +167,7 @@ mod benchtool { }, None => host.push_str("2003"), } - let rand = thread_rng(); + let mut rand = thread_rng(); if let Some(matches) = matches.subcommand_matches("testkey") { let numkeys = matches.value_of("count").unwrap(); if let Ok(num) = numkeys.parse::() { @@ -175,17 +175,11 @@ mod benchtool { println!("Generating keys ..."); let keys: Vec = (0..num) .into_iter() - .map(|_| { - let rand_string: String = rand.sample_iter(&Alphanumeric).take(8).collect(); - rand_string - }) + .map(|_| ran_string(8, &mut rand)) .collect(); let values: Vec = (0..num) .into_iter() - .map(|_| { - let rand_string: String = rand.sample_iter(&Alphanumeric).take(8).collect(); - rand_string - }) + .map(|_| ran_string(8, &mut rand)) .collect(); let set_packs: Vec> = (0..num) .map(|idx| terrapipe::proc_query(format!("SET {} {}", keys[idx], values[idx]))) @@ -225,26 +219,18 @@ mod benchtool { max_queries, (packet_size * 2), // key size + value size ); - let rand = thread_rng(); + let mut rand = thread_rng(); let mut dt = DevTime::new_complex(); // Create separate connection pools for get and set operations let mut setpool = Netpool::new(max_connections, &host); let mut getpool = Netpool::new(max_connections, &host); let keys: Vec = (0..max_queries) .into_iter() - .map(|_| { - let rand_string: String = - rand.sample_iter(&Alphanumeric).take(packet_size).collect(); - rand_string - }) + .map(|_| ran_string(packet_size, &mut rand)) .collect(); let values: Vec = (0..max_queries) .into_iter() - .map(|_| { - let rand_string: String = - rand.sample_iter(&Alphanumeric).take(packet_size).collect(); - rand_string - }) + .map(|_| ran_string(packet_size, &mut rand)) .collect(); /* We create three vectors of vectors: `set_packs`, `get_packs` and `del_packs` @@ -309,6 +295,15 @@ mod benchtool { fn calc(reqs: usize, time: u128) -> f64 { reqs as f64 / (time as f64 / 1_000_000_000 as f64) } + + fn ran_string(len: usize, rand: impl rand::Rng) -> String { + let rand_string: String = rand + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect(); + rand_string + } } fn main() { diff --git a/tdb-macros/Cargo.toml b/tdb-macros/Cargo.toml index c0a39227..7f467499 100644 --- a/tdb-macros/Cargo.toml +++ b/tdb-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tdb_macros" -version = "0.2.2" +version = "0.5.0" authors = ["Sayan Nandan "] edition = "2018" @@ -12,5 +12,5 @@ proc-macro = true [dependencies] syn = {version = "1.0.58", features = ["full"]} quote = "1.0.8" -rand = "0.8.1" +rand = "0.8.2" proc-macro2 = "1.0.24"