From 8e429c17f4b17adb31386f42eee1feeffe7149bf Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Fri, 2 Oct 2020 18:13:43 +0530 Subject: [PATCH] Add basic `Snapshot` object --- Cargo.lock | 58 +++++++++++++++++++++++++++++- server/Cargo.toml | 1 + server/src/coredb/mod.rs | 5 +-- server/src/diskstore/mod.rs | 6 ++-- server/src/diskstore/snapshot.rs | 60 ++++++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 server/src/diskstore/snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index 24f3cf84..d27a1753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + [[package]] name = "bincode" version = "1.3.1" @@ -75,6 +81,19 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi 0.3.9", +] + [[package]] name = "clap" version = "2.33.3" @@ -161,7 +180,7 @@ checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] @@ -348,6 +367,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.0" @@ -568,6 +606,7 @@ version = "0.4.3" dependencies = [ "bincode", "bytes", + "chrono", "clap", "env_logger", "jemallocator", @@ -632,6 +671,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + [[package]] name = "tokio" version = "0.2.22" @@ -711,6 +761,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "winapi" version = "0.2.8" diff --git a/server/Cargo.toml b/server/Cargo.toml index 83749d04..cf6420c8 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -19,6 +19,7 @@ toml = "0.5.6" clap = {version = "2.33.3", features=["yaml"]} env_logger = "0.7.1" log = "0.4.11" +chrono = "0.4.19" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.3.2" diff --git a/server/src/coredb/mod.rs b/server/src/coredb/mod.rs index 1622072f..4103cb86 100644 --- a/server/src/coredb/mod.rs +++ b/server/src/coredb/mod.rs @@ -27,6 +27,7 @@ use crate::protocol::Connection; use crate::protocol::Query; use crate::queryengine; use bytes::Bytes; +use diskstore::PERSIST_FILE; use libtdb::TResult; use parking_lot::RwLock; use parking_lot::RwLockReadGuard; @@ -67,7 +68,7 @@ impl Shared { return None; } // Kick in BGSAVE - match diskstore::flush_data(&self.table.read().get_ref()) { + match diskstore::flush_data(PERSIST_FILE, &self.table.read().get_ref()) { Ok(_) => log::info!("BGSAVE completed successfully"), Err(e) => log::error!("BGSAVE failed with error: '{}'", e), } @@ -189,7 +190,7 @@ impl CoreDB { /// Flush the contents of the in-memory table onto disk pub fn flush_db(&self) -> TResult<()> { let data = &self.acquire_write(); - diskstore::flush_data(&data.coremap)?; + diskstore::flush_data(PERSIST_FILE, &data.coremap)?; Ok(()) } diff --git a/server/src/diskstore/mod.rs b/server/src/diskstore/mod.rs index 5ad203ad..b3a7a375 100644 --- a/server/src/diskstore/mod.rs +++ b/server/src/diskstore/mod.rs @@ -33,8 +33,10 @@ use std::iter::FromIterator; use std::time::Duration; use tokio::time; mod cyansfw; +mod snapshot; type DiskStore = (Vec, Vec>); +pub const PERSIST_FILE: &'static str = "./data.bin"; /// Try to get the saved data from disk. This returns `None`, if the `data.bin` wasn't found /// otherwise the `data.bin` file is deserialized and parsed into a `HashMap` @@ -64,13 +66,13 @@ pub fn get_saved() -> TResult>> { /// /// This functions takes the entire in-memory table and writes it to the disk, /// more specifically, the `data.bin` file -pub fn flush_data(data: &HashMap) -> TResult<()> { +pub fn flush_data(filename: &str, data: &HashMap) -> TResult<()> { let ds: DiskStore = ( data.keys().into_iter().map(|val| val.to_string()).collect(), data.values().map(|val| val.get_blob().to_vec()).collect(), ); let encoded = bincode::serialize(&ds)?; - let mut file = fs::File::create("./data.bin")?; + let mut file = fs::File::create(filename)?; file.write_all(&encoded)?; Ok(()) } diff --git a/server/src/diskstore/snapshot.rs b/server/src/diskstore/snapshot.rs new file mode 100644 index 00000000..7056968b --- /dev/null +++ b/server/src/diskstore/snapshot.rs @@ -0,0 +1,60 @@ +/* + * Created on Thu Oct 01 2020 + * + * This file is a part of TerrabaseDB + * Copyright (c) 2020, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +//! Tools for creating snapshots + +use crate::coredb::CoreDB; +use chrono::prelude::*; +use libtdb::TResult; + +/// # Snapshot +/// +/// This object provides methods to create and delete snapshots. There should be a +/// `snapshot_scheduler` which should hold an instance of this object, on startup. +/// Whenever the duration expires, the caller should call `mksnap()` +pub struct Snapshot { + /// File names of the snapshots (relative paths) + snaps: Vec, + /// The maximum number of snapshots to be kept + maxtop: usize, + /// An atomic reference to the coretable + dbref: CoreDB, +} + +impl Snapshot { + /// Create a new `Snapshot` instance + pub fn new(maxtop: usize, dbref: CoreDB) -> Self { + Snapshot { + snaps: Vec::with_capacity(maxtop), + maxtop, + dbref, + } + } + /// Generate the snapshot name + fn get_snapname(&self) -> String { + Utc::now() + .format("./snapshots/%Y%m%d-%H%M%S.snapshot") + .to_string() + } + pub fn mksnap(&mut self) -> TResult<()> { + todo!("Snapshotting hasn't been implemented yet!") + } +}