From 2160ae37cf7a3914d694d94014e08a890ba28900 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Tue, 17 Aug 2021 06:07:30 -0700 Subject: [PATCH] Add migration tool Closes #200 --- Cargo.lock | 26 ++++++-- Cargo.toml | 1 + Makefile | 10 +-- sky-migrate/Cargo.toml | 14 +++++ sky-migrate/src/cli.yml | 30 +++++++++ sky-migrate/src/main.rs | 135 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 206 insertions(+), 10 deletions(-) create mode 100644 sky-migrate/Cargo.toml create mode 100644 sky-migrate/src/cli.yml create mode 100644 sky-migrate/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 0762ef64..8d3e3e5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,7 +472,7 @@ version = "0.7.0" dependencies = [ "lazy_static", "regex", - "skytable", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust?branch=next)", "termcolor", ] @@ -943,7 +943,18 @@ dependencies = [ "rand", "serde", "serde_json", - "skytable", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust?branch=next)", +] + +[[package]] +name = "sky-migrate" +version = "0.1.0" +dependencies = [ + "bincode", + "clap", + "env_logger", + "log", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust.git)", ] [[package]] @@ -980,7 +991,7 @@ dependencies = [ "regex", "serde", "sky_macros", - "skytable", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust?branch=next)", "tokio", "tokio-openssl", "toml", @@ -995,7 +1006,7 @@ dependencies = [ "crossterm", "libsky", "rustyline", - "skytable", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust?branch=next)", "tokio", ] @@ -1010,6 +1021,11 @@ dependencies = [ "tokio-openssl", ] +[[package]] +name = "skytable" +version = "0.4.0" +source = "git+https://github.com/skytable/client-rust.git#376c05e7ae83ae1c0c06ec10ef0039774819f6c7" + [[package]] name = "slab" version = "0.4.4" @@ -1039,7 +1055,7 @@ dependencies = [ "log", "num_cpus", "rand", - "skytable", + "skytable 0.4.0 (git+https://github.com/skytable/client-rust?branch=next)", "sysinfo", ] diff --git a/Cargo.toml b/Cargo.toml index 049be9f9..6e892dc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "sky-macros", "libstress", "stress-test", + "sky-migrate", ] [profile.release] diff --git a/Makefile b/Makefile index a8399510..5696215e 100644 --- a/Makefile +++ b/Makefile @@ -73,25 +73,25 @@ ifeq ($(origin TARGET),undefined) ifeq ($(OS),Windows_NT) # windows, so we need exe BUNDLE += cd target/release && -BUNDLE += 7z a ../../../bundle.zip skysh.exe skyd.exe sky-bench.exe +BUNDLE += 7z a ../../../bundle.zip skysh.exe skyd.exe sky-bench.exe sky-migrate.exe else # not windows, so no exe -BUNDLE+=zip -j bundle.zip target/release/skysh target/release/skyd target/release/sky-bench +BUNDLE+=zip -j bundle.zip target/release/skysh target/release/skyd target/release/sky-bench target/release/sky-migrate endif else # target was defined, but check for windows ifeq ($(OS),Windows_NT) # windows, so we need exe BUNDLE += cd target/${TARGET}/release && -BUNDLE+=7z a ../../../sky-bundle-${VERSION}-${ARTIFACT}.zip skysh.exe skyd.exe sky-bench.exe +BUNDLE+=7z a ../../../sky-bundle-${VERSION}-${ARTIFACT}.zip skysh.exe skyd.exe sky-bench.exe sky-migrate.exe else # not windows, so no exe ifneq ($(origin CARGO_TARGET_DIR),undefined) # target defined and target dir. use this instead of target/ -BUNDLE+=zip -j sky-bundle-${VERSION}-${ARTIFACT}.zip ${CARGO_TARGET_DIR}/${TARGET}/release/skysh ${CARGO_TARGET_DIR}/${TARGET}/release/skyd ${CARGO_TARGET_DIR}/${TARGET}/release/sky-bench +BUNDLE+=zip -j sky-bundle-${VERSION}-${ARTIFACT}.zip ${CARGO_TARGET_DIR}/${TARGET}/release/skysh ${CARGO_TARGET_DIR}/${TARGET}/release/skyd ${CARGO_TARGET_DIR}/${TARGET}/release/sky-bench ${CARGO_TARGET_DIR}/${TARGET}/release/sky-migrate else # just the plain old target/${TARGET} path -BUNDLE+=zip -j sky-bundle-${VERSION}-${ARTIFACT}.zip target/${TARGET}/release/skysh target/${TARGET}/release/skyd target/${TARGET}/release/sky-bench +BUNDLE+=zip -j sky-bundle-${VERSION}-${ARTIFACT}.zip target/${TARGET}/release/skysh target/${TARGET}/release/skyd target/${TARGET}/release/sky-bench target/${TARGET}/release/sky-migrate endif endif endif diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml new file mode 100644 index 00000000..637f7718 --- /dev/null +++ b/sky-migrate/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sky-migrate" +version = "0.1.0" +authors = ["Sayan Nandan "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +skytable = { git = "https://github.com/skytable/client-rust.git" } +env_logger = "0.9.0" +bincode = "1.3.3" +log = "0.4.14" +clap = { version = "2.33.3", features = ["yaml"] } diff --git a/sky-migrate/src/cli.yml b/sky-migrate/src/cli.yml new file mode 100644 index 00000000..6f9b0302 --- /dev/null +++ b/sky-migrate/src/cli.yml @@ -0,0 +1,30 @@ +name: Skytable Migration Tool +version: 0.7.0 +author: Sayan N. +about: | + The Skytable migration tool allows users coming from older versions (>=0.6.0) + to upgrade their datasets to the latest Skytable version. This tool currently + supports versions >= 0.6.0 and upgrading it to 0.7.0. To upgrade, on needs + to simply run: + sky-migrate --prevdir --new : + Where `` is the path to the last installation's data directory and + `` and `` is the hostname and port for the new server instance +args: + - new: + long: new + takes_value: true + required: true + help: The : combo for the new instance + value_name: new + - prevdir: + long: prevdir + takes_value: true + required: true + help: Path to the previous installation location + value_name: prevdir + - serial: + long: serial + takes_value: false + required: false + help: | + Transfer entries one-by-one instead of all at once to save memory diff --git a/sky-migrate/src/main.rs b/sky-migrate/src/main.rs new file mode 100644 index 00000000..85303bb1 --- /dev/null +++ b/sky-migrate/src/main.rs @@ -0,0 +1,135 @@ +/* + * Created on Tue Aug 17 2021 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2021, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![allow(clippy::unit_arg)] + +use clap::{load_yaml, App}; +use core::hint::unreachable_unchecked; +use env_logger::Builder; +use log::error as err; +use log::info; +use skytable::query; +use skytable::sync::Connection; +use skytable::Query; +use skytable::RespCode; +use skytable::{Element, Response}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process; +type Bytes = Vec; + +fn main() { + // first evaluate config + let cfg_layout = load_yaml!("cli.yml"); + let matches = App::from_yaml(cfg_layout).get_matches(); + Builder::new() + .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned())) + .init(); + let new_host = matches + .value_of("new") + .map(|v| v.to_string()) + .unwrap_or_else(|| unsafe { unreachable_unchecked() }); + let serial = matches.is_present("serial"); + let hostsplit: Vec<&str> = new_host.split(':').collect(); + if hostsplit.len() != 2 { + err(err!("Bad value for --new")); + } + let (host, port) = unsafe { (hostsplit.get_unchecked(0), hostsplit.get_unchecked(1)) }; + let port = match port.parse() { + Ok(p) => p, + Err(e) => err(err!("Bad value for port in --new: {}", e)), + }; + let mut old_dir = matches + .value_of("prevdir") + .map(PathBuf::from) + .unwrap_or_else(|| unsafe { unreachable_unchecked() }); + old_dir.push("data.bin"); + // now connect + let mut con = match Connection::new(host, port) { + Ok(con) => con, + Err(e) => err(err!("Failed to connect to new instance with error: {}", e)), + }; + // run sanity test + let q = query!("HEYA"); + match con.run_simple_query(&q) { + Ok(Response::Item(Element::Str(s))) if s.eq("HEY!") => {} + Ok(_) => err(err!("Unknown response from server")), + Err(e) => err(err!( + "An I/O error occurred while running sanity test: {}", + e + )), + } + info!("Sanity test complete"); + + // now de old file + let read = match fs::read(old_dir) { + Ok(r) => r, + Err(e) => err(err!( + "Failed to read data.bin file from old directory: {}", + e + )), + }; + let de: HashMap = match bincode::deserialize(&read) { + Ok(r) => r, + Err(e) => err(err!("Failed to unpack old file with: {}", e)), + }; + unsafe { + if serial { + // transfer serially + for (key, value) in de.into_iter() { + let q = query!( + "USET", + String::from_utf8_unchecked(key), + String::from_utf8_unchecked(value) + ); + okay(&mut con, q) + } + } else { + // transfer all at once + let mut query = Query::from("USET"); + for (key, value) in de.into_iter() { + query.push(String::from_utf8_unchecked(key)); + query.push(String::from_utf8_unchecked(value)); + } + okay(&mut con, query) + } + } + info!("Finished migration"); +} + +fn err(_i: ()) -> ! { + process::exit(0x01) +} + +fn okay(con: &mut Connection, q: Query) { + match con.run_simple_query(&q) { + Ok(Response::Item(Element::RespCode(RespCode::Okay))) => {} + Err(e) => err(err!("An I/O error occurred while running query: {}", e)), + Ok(_) => err(err!("Unknown response from server")), + } +}