Improve compat to use storage formats for upgrades (#158)

server/src/cli.yml

@@ -81,4 +81,12 @@ args:
           help: Tells the server to only accept SSL connections and disables the non-SSL port
 subcommands:
       - upgrade:
-            about: Upgrades old datsets to the latest format supported by this server edition
+            about: Upgrades old datsets to the latest format supported by this server edition
+            args:
+                  - format:
+                        short: f
+                        long: from
+                        takes_value: true
+                        value_name: format
+                        required: true
+                        help: The format of the old files which need to be upgraded

server/src/compat/mod.rs

@@ -32,6 +32,7 @@
 use crate::coredb::{htable::HTable, Data};
 use crate::diskstore::snapshot::SNAP_MATCH;
 use bytes::Bytes;
+use core::hint::unreachable_unchecked;
 use libsky::TResult;
 use std::collections::HashMap;
 use std::fs;
@@ -45,57 +46,93 @@ type DiskStoreType = (Vec<String>, Vec<Vec<u8>>);
 const SKY_UPGRADE_FOLDER: &str = "newdata";
 const SKY_COMPLETE_UPGRADE_FOLDER: &str = "newdata/snapshots/remote";
 
+enum Format {
+    /// The disk storage format used in 0.3.0
+    Elstore,
+    /// The disk storage format used between 0.3.1-0.5.2
+    Neocopy,
+    /// The disk storage format used in 0.6.0
+    Sparrowlock,
+}
+
+impl Format {
+    pub const fn has_snapshots(&self) -> bool {
+        if let Self::Neocopy | Self::Sparrowlock = self {
+            true
+        } else {
+            false
+        }
+    }
+}
+
 pub fn concat_path(other: impl Into<PathBuf>) -> PathBuf {
     let mut path = PathBuf::from(SKY_UPGRADE_FOLDER);
     path.push(other.into());
     path
 }
 
-pub fn upgrade() -> TResult<()> {
+pub fn upgrade(format: &str) -> TResult<()> {
+    let fmt = match format {
+        "elstore" => Format::Elstore,
+        "neocopy" => Format::Neocopy,
+        "sparrowlock" => Format::Sparrowlock,
+        _ => return Err("Unknown format".into()),
+    };
+    if let Format::Sparrowlock = fmt {
+        log::info!("No file upgrades required");
+        return Ok(());
+    }
     fs::create_dir_all(SKY_COMPLETE_UPGRADE_FOLDER)?;
     // first attempt to upgrade the data file
     log::info!("Upgrading data file");
-    upgrade_file("data/data.bin", concat_path("data.bin"))
+    upgrade_file("data/data.bin", concat_path("data.bin"), &fmt)
         .map_err(|e| format!("Failed to upgrade data.bin file with error: {}", e))?;
     log::info!("Finished upgrading data file");
     // now let's check what files are there in the snapshots directory
-    log::info!("Upgrading snapshots");
+    if fmt.has_snapshots() {
-    let snapshot_dir = fs::read_dir("data/snapshots")?;
+        log::info!("Upgrading snapshots");
-    for path in snapshot_dir {
+        let snapshot_dir = fs::read_dir("data/snapshots")?;
-        let path = path?.path();
+        for path in snapshot_dir {
-        if path.is_dir() && path != PathBuf::from("data/snapshots/remote") {
+            let path = path?.path();
-            return Err("The snapshot directory contains unrecognized files".into());
+            if path.is_dir() && path != PathBuf::from("data/snapshots/remote") {
-        }
+                return Err("The snapshot directory contains unrecognized files".into());
-        if path.is_file() {
+            }
-            let fname = path
+            if path.is_file() {
-                .file_name()
+                let fname = path
-                .ok_or("Failed to get path name in snapshot directory")?
+                    .file_name()
-                .to_string_lossy();
+                    .ok_or("Failed to get path name in snapshot directory")?
-            if !SNAP_MATCH.is_match(&fname) {
+                    .to_string_lossy();
-                return Err("The snapshot directory contains unexpected files".into());
+                if !SNAP_MATCH.is_match(&fname) {
+                    return Err("The snapshot directory contains unexpected files".into());
+                }
+                upgrade_file(
+                    path.clone(),
+                    concat_path(format!("snapshots/{}", fname)),
+                    &fmt,
+                )?;
             }
-            upgrade_file(path.clone(), concat_path(format!("snapshots/{}", fname)))?;
         }
-    }
+        log::info!("Finished upgrading snapshots");
-    log::info!("Finished upgrading snapshots");
+        log::info!("Upgrading remote snapshots");
-    log::info!("Upgrading remote snapshots");
+        let remote_snapshot_dir = fs::read_dir("data/snapshots/remote")?;
-    let remote_snapshot_dir = fs::read_dir("data/snapshots/remote")?;
+        for path in remote_snapshot_dir {
-    for path in remote_snapshot_dir {
+            let path = path?.path();
-        let path = path?.path();
+            if path.is_file() {
-        if path.is_file() {
+                let fname = path
-            let fname = path
+                    .file_name()
-                .file_name()
+                    .ok_or("Failed to get filename in remote snapshot directory")?
-                .ok_or("Failed to get filename in remote snapshot directory")?
+                    .to_string_lossy();
-                .to_string_lossy();
+                upgrade_file(
-            upgrade_file(
+                    path.clone(),
-                path.clone(),
+                    concat_path(format!("snapshots/remote/{}", fname)),
-                concat_path(format!("snapshots/remote/{}", fname)),
+                    &fmt,
-            )?;
+                )?;
-        } else {
+            } else {
-            return Err("Unexpected files in the remote snapshot directory".into());
+                return Err("Unexpected files in the remote snapshot directory".into());
+            }
         }
+        log::info!("Finished upgrading remote snapshots");
     }
-    log::info!("Finished upgrading remote snapshots");
     log::info!("All files were upgraded. Updating directories");
     fs::rename("data", "olddata")?;
     log::info!("Moved old data into folder 'olddata'");
@@ -104,23 +141,42 @@ pub fn upgrade() -> TResult<()> {
     Ok(())
 }
 
-fn upgrade_file(src: impl Into<PathBuf>, destination: impl Into<PathBuf>) -> TResult<()> {
+fn upgrade_file(
+    src: impl Into<PathBuf>,
+    destination: impl Into<PathBuf>,
+    fmt: &Format,
+) -> TResult<()> {
     let file = src.into();
     log::info!("Upgrading file: {}", file.to_string_lossy());
     let old_data_file = fs::read(&file)?;
-    let data_from_old_file: DiskStoreType = bincode::deserialize(&old_data_file)?;
+    let data_in_new_format: HTable<Data, Data> = match *fmt {
-    let data_from_old_file: HashMap<String, Data> = HashMap::from_iter(
+        Format::Elstore => {
-        data_from_old_file
+            let data_from_old_file: HashMap<String, String> = bincode::deserialize(&old_data_file)?;
-            .0
+            HTable::from_iter(
-            .into_iter()
+                data_from_old_file
-            .zip(data_from_old_file.1.into_iter())
+                    .into_iter()
-            .map(|(key, value)| (key, Data::from_blob(Bytes::from(value)))),
+                    .map(|(key, value)| (Data::from(key), Data::from(value))),
-    );
+            )
-    let data_in_new_format: HTable<Data, Data> = HTable::from_iter(
+        }
-        data_from_old_file
+        Format::Neocopy => {
-            .into_iter()
+            let data_from_old_file: DiskStoreType = bincode::deserialize(&old_data_file)?;
-            .map(|(key, value)| (Data::from_string(key), value)),
+            let data_from_old_file: HashMap<String, Data> = HashMap::from_iter(
-    );
+                data_from_old_file
+                    .0
+                    .into_iter()
+                    .zip(data_from_old_file.1.into_iter())
+                    .map(|(key, value)| (key, Data::from_blob(Bytes::from(value)))),
+            );
+            HTable::from_iter(
+                data_from_old_file
+                    .into_iter()
+                    .map(|(key, value)| (Data::from_string(key), value)),
+            )
+        }
+        Format::Sparrowlock => unsafe {
+            unreachable_unchecked();
+        },
+    };
     let data_in_new_format = data_in_new_format.serialize()?;
     let destination = destination.into();
     let mut file = fs::File::create(&destination)?;

server/src/config/mod.rs

@@ -27,6 +27,7 @@
 //! This module provides tools to handle configuration files and settings
 
 use crate::compat;
+use core::hint::unreachable_unchecked;
 #[cfg(test)]
 use libsky::TResult;
 use serde::Deserialize;
@@ -411,12 +412,18 @@ pub fn get_config_file_or_return_cfg() -> Result<ConfigType<ParsedConfig, String
     let cfg_layout = load_yaml!("../cli.yml");
     let matches = App::from_yaml(cfg_layout).get_matches();
     // check upgrades
-    if matches.subcommand_matches("upgrade").is_some() {
+    if let Some(matches) = matches.subcommand_matches("upgrade") {
-        if let Err(e) = compat::upgrade() {
+        if let Some(format) = matches.value_of("format") {
-            log::error!("Dataset upgrade failed with error: {}", e);
+            if let Err(e) = compat::upgrade(format) {
-            process::exit(0x100);
+                log::error!("Dataset upgrade failed with error: {}", e);
+                process::exit(0x100);
+            } else {
+                process::exit(0x000);
+            }
         } else {
-            process::exit(0x000);
+            unsafe {
+                unreachable_unchecked();
+            }
         }
     }
     let restorefile = matches.value_of("restore").map(|v| v.to_string());