Implement and refactor to use common storage interfaces
Sayan Nandan
8 months ago
parent
3f2b464975
commit
f769175083
@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Created on Sun Sep 03 2023
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
//! # Checksum utils
|
||||
//!
|
||||
//! This module contains utils for handling checksums
|
||||
//!
|
||||
|
||||
use crc::{Crc, Digest, CRC_64_XZ};
|
||||
|
||||
/*
|
||||
NOTE(@ohsayan): we're currently using crc's impl. but the reason I decided to make a wrapper is because I have a
|
||||
different impl in mind
|
||||
*/
|
||||
|
||||
const CRC64: Crc<u64> = Crc::<u64>::new(&CRC_64_XZ);
|
||||
|
||||
pub struct SCrc64 {
|
||||
digest: Digest<'static, u64>,
|
||||
}
|
||||
|
||||
impl SCrc64 {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
digest: CRC64.digest(),
|
||||
}
|
||||
}
|
||||
pub fn recompute_with_new_var_block(&mut self, b: &[u8]) {
|
||||
self.digest.update(b)
|
||||
}
|
||||
pub fn finish(self) -> u64 {
|
||||
self.digest.finalize()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Created on Sun Jan 07 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
use {
|
||||
super::fs_traits::{
|
||||
FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, FileInterfaceRead,
|
||||
FileInterfaceWrite, FileInterfaceWriteExt, FileOpen,
|
||||
},
|
||||
crate::engine::RuntimeResult,
|
||||
std::{
|
||||
fs::{self, File},
|
||||
io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
|
||||
},
|
||||
};
|
||||
|
||||
/*
|
||||
local fs impls
|
||||
*/
|
||||
|
||||
/// A type representing the host's local filesystem (or atleast where our data directory is)
|
||||
pub struct LocalFS;
|
||||
|
||||
fn cvt<T, E1, E2: From<E1>>(r: Result<T, E1>) -> Result<T, E2> {
|
||||
r.map_err(Into::into)
|
||||
}
|
||||
|
||||
impl FSInterface for LocalFS {
|
||||
type File = File;
|
||||
fn fs_remove_file(fpath: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::remove_file(fpath))
|
||||
}
|
||||
fn fs_rename(from: &str, to: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::rename(from, to))
|
||||
}
|
||||
fn fs_create_dir(fpath: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::create_dir(fpath))
|
||||
}
|
||||
fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::create_dir_all(fpath))
|
||||
}
|
||||
fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::remove_dir(fpath))
|
||||
}
|
||||
fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> {
|
||||
cvt(fs::remove_dir_all(fpath))
|
||||
}
|
||||
fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<super::fs_traits::FileOpen<Self::File>> {
|
||||
let r = || -> Result<_, std::io::Error> {
|
||||
let f = File::options()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(fpath)?;
|
||||
let md = f.metadata()?;
|
||||
if md.len() == 0 {
|
||||
Ok(FileOpen::Created(f))
|
||||
} else {
|
||||
Ok(FileOpen::Existing(f))
|
||||
}
|
||||
};
|
||||
cvt(r())
|
||||
}
|
||||
fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
||||
let f = File::options().read(true).write(true).open(fpath)?;
|
||||
Ok(f)
|
||||
}
|
||||
fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
||||
let f = File::options()
|
||||
.create_new(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(fpath)?;
|
||||
Ok(f)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
common impls for files
|
||||
*/
|
||||
|
||||
impl<R: Read> FileInterfaceRead for R {
|
||||
fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
|
||||
cvt(self.read_exact(buf))
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> FileInterfaceWrite for W {
|
||||
fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult<u64> {
|
||||
cvt(self.write(buf).map(|v| v as _))
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
local file impls
|
||||
*/
|
||||
|
||||
impl FileInterface for File {
|
||||
type BufReader = BufReader<Self>;
|
||||
type BufWriter = BufWriter<Self>;
|
||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
|
||||
Ok(BufReader::new(self))
|
||||
}
|
||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
|
||||
Ok(BufWriter::new(self))
|
||||
}
|
||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
|
||||
Ok(r.into_inner())
|
||||
}
|
||||
fn downgrade_writer(mut r: Self::BufWriter) -> RuntimeResult<Self> {
|
||||
// TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this?
|
||||
r.flush()?;
|
||||
let (me, err) = r.into_parts();
|
||||
match err {
|
||||
Ok(x) if x.is_empty() => Ok(me),
|
||||
Ok(_) | Err(_) => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"failed to flush data from buffer into sink",
|
||||
)
|
||||
.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A trait for handling wrappers of [`std::fs::File`]
|
||||
trait AsLocalFile {
|
||||
fn file(&self) -> &File;
|
||||
fn file_mut(&mut self) -> &mut File;
|
||||
}
|
||||
|
||||
impl AsLocalFile for File {
|
||||
fn file(&self) -> &File {
|
||||
self
|
||||
}
|
||||
fn file_mut(&mut self) -> &mut File {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl AsLocalFile for BufReader<File> {
|
||||
fn file(&self) -> &File {
|
||||
self.get_ref()
|
||||
}
|
||||
fn file_mut(&mut self) -> &mut File {
|
||||
self.get_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl AsLocalFile for BufWriter<File> {
|
||||
fn file(&self) -> &File {
|
||||
self.get_ref()
|
||||
}
|
||||
fn file_mut(&mut self) -> &mut File {
|
||||
self.get_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceBufWrite for BufWriter<File> {
|
||||
fn sync_write_cache(&mut self) -> RuntimeResult<()> {
|
||||
// TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this?
|
||||
cvt(self.flush())
|
||||
}
|
||||
}
|
||||
|
||||
impl<F: AsLocalFile> FileInterfaceExt for F {
|
||||
fn fext_length(&self) -> RuntimeResult<u64> {
|
||||
Ok(self.file().metadata()?.len())
|
||||
}
|
||||
fn fext_cursor(&mut self) -> RuntimeResult<u64> {
|
||||
cvt(self.file_mut().stream_position())
|
||||
}
|
||||
fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> {
|
||||
cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceWriteExt for File {
|
||||
fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
|
||||
cvt(self.set_len(to))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,607 @@
|
||||
/*
|
||||
* Created on Sun Jan 07 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
//! File system emulation
|
||||
//!
|
||||
//! This directory contains some implementations of virtual file systems (either emulating `/tmp` or
|
||||
//! `/dev/null`) that are directly implemented at the application level with some necessary changes
|
||||
//! required for testing
|
||||
//!
|
||||
|
||||
use {
|
||||
super::fs_traits::{
|
||||
FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, FileInterfaceRead,
|
||||
FileInterfaceWrite, FileInterfaceWriteExt, FileOpen,
|
||||
},
|
||||
crate::engine::{sync::cell::Lazy, RuntimeResult},
|
||||
parking_lot::RwLock,
|
||||
std::{
|
||||
collections::{
|
||||
hash_map::{Entry, OccupiedEntry},
|
||||
HashMap,
|
||||
},
|
||||
io::{Error, ErrorKind},
|
||||
},
|
||||
};
|
||||
|
||||
/*
|
||||
vfs definitions
|
||||
*/
|
||||
|
||||
/// # VirtualFS
|
||||
///
|
||||
/// A virtual file system stored entirely in the process's memory (inclusive of swap if enabled; no explicit discrimination is made)
|
||||
///
|
||||
/// The virtual file system is generally intended for being utilized as an in-memory filesystem, primarily for testing
|
||||
/// purposes and has a lot of limitations.
|
||||
///
|
||||
/// It has support for:
|
||||
/// - nested directories
|
||||
/// - read/write permissions
|
||||
/// - file position tracking and seeking
|
||||
/// - directory operations
|
||||
pub struct VirtualFS;
|
||||
|
||||
/// A virtual directory
|
||||
type VDir = HashMap<Box<str>, VNode>;
|
||||
/// An iterator over the components of a file path (alias)
|
||||
type ComponentIter<'a> = std::iter::Take<std::vec::IntoIter<&'a str>>;
|
||||
|
||||
/**
|
||||
vnode
|
||||
---
|
||||
either a vfile or a vdir
|
||||
*/
|
||||
#[derive(Debug)]
|
||||
pub(super) enum VNode {
|
||||
Dir(HashMap<Box<str>, Self>),
|
||||
File(VFile),
|
||||
}
|
||||
|
||||
impl VNode {
|
||||
fn as_dir_mut(&mut self) -> Option<&mut VDir> {
|
||||
match self {
|
||||
Self::Dir(d) => Some(d),
|
||||
Self::File(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
vfile
|
||||
*/
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VFile {
|
||||
read: bool,
|
||||
write: bool,
|
||||
data: Vec<u8>,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl VFile {
|
||||
fn new(read: bool, write: bool, data: Vec<u8>, pos: usize) -> Self {
|
||||
Self {
|
||||
read,
|
||||
write,
|
||||
data,
|
||||
pos,
|
||||
}
|
||||
}
|
||||
fn current(&self) -> &[u8] {
|
||||
&self.data[self.pos..]
|
||||
}
|
||||
}
|
||||
|
||||
mod err {
|
||||
//! Errors
|
||||
//!
|
||||
//! These are custom errors returned by the virtual file system
|
||||
use {
|
||||
crate::engine::RuntimeResult,
|
||||
std::io::{Error, ErrorKind},
|
||||
};
|
||||
pub(super) fn item_is_not_file<T>() -> RuntimeResult<T> {
|
||||
Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
|
||||
}
|
||||
pub(super) fn file_in_dir_path<T>() -> RuntimeResult<T> {
|
||||
Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into())
|
||||
}
|
||||
pub(super) fn dir_missing_in_path<T>() -> RuntimeResult<T> {
|
||||
Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into())
|
||||
}
|
||||
pub(super) fn could_not_find_item<T>() -> RuntimeResult<T> {
|
||||
Err(Error::new(ErrorKind::NotFound, "could not find item").into())
|
||||
}
|
||||
}
|
||||
|
||||
mod util {
|
||||
use {
|
||||
super::{err, ComponentIter, VDir, VNode},
|
||||
crate::engine::RuntimeResult,
|
||||
};
|
||||
pub(super) fn split_parts(fpath: &str) -> Vec<&str> {
|
||||
fpath.split("/").collect()
|
||||
}
|
||||
pub(super) fn split_target_and_components(fpath: &str) -> (&str, ComponentIter) {
|
||||
let parts = split_parts(fpath);
|
||||
let target = parts.last().unwrap();
|
||||
let component_len = parts.len() - 1;
|
||||
(target, parts.into_iter().take(component_len))
|
||||
}
|
||||
pub(super) fn find_target_dir_mut<'a>(
|
||||
components: ComponentIter,
|
||||
mut current: &'a mut VDir,
|
||||
) -> RuntimeResult<&'a mut VDir> {
|
||||
for component in components {
|
||||
match current.get_mut(component) {
|
||||
Some(VNode::Dir(d)) => current = d,
|
||||
Some(VNode::File(_)) => return err::file_in_dir_path(),
|
||||
None => return err::dir_missing_in_path(),
|
||||
}
|
||||
}
|
||||
Ok(current)
|
||||
}
|
||||
pub(super) fn find_target_dir<'a>(
|
||||
components: ComponentIter,
|
||||
mut current: &'a VDir,
|
||||
) -> RuntimeResult<&'a VDir> {
|
||||
for component in components {
|
||||
match current.get(component) {
|
||||
Some(VNode::Dir(d)) => current = d,
|
||||
Some(VNode::File(_)) => return err::file_in_dir_path(),
|
||||
None => return err::dir_missing_in_path(),
|
||||
}
|
||||
}
|
||||
Ok(current)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
vfs impl:
|
||||
- nested directory structure
|
||||
- make parents
|
||||
- make child
|
||||
*/
|
||||
|
||||
impl VirtualFS {
|
||||
/// Get a handle to the virtual filesystem
|
||||
fn handle() -> &'static RwLock<VDir> {
|
||||
static VFS: Lazy<RwLock<VDir>, fn() -> RwLock<VDir>> = Lazy::new(|| Default::default());
|
||||
&VFS
|
||||
}
|
||||
fn with_file_mut<T>(
|
||||
fpath: &str,
|
||||
mut f: impl FnMut(&mut VFile) -> RuntimeResult<T>,
|
||||
) -> RuntimeResult<T> {
|
||||
let mut vfs = Self::handle().write();
|
||||
let (target_file, components) = util::split_target_and_components(fpath);
|
||||
let target_dir = util::find_target_dir_mut(components, &mut vfs)?;
|
||||
match target_dir.get_mut(target_file) {
|
||||
Some(VNode::File(file)) => f(file),
|
||||
Some(VNode::Dir(_)) => return err::item_is_not_file(),
|
||||
None => return Err(Error::from(ErrorKind::NotFound).into()),
|
||||
}
|
||||
}
|
||||
fn with_file<T>(
|
||||
fpath: &str,
|
||||
mut f: impl FnMut(&VFile) -> RuntimeResult<T>,
|
||||
) -> RuntimeResult<T> {
|
||||
let vfs = Self::handle().read();
|
||||
let (target_file, components) = util::split_target_and_components(fpath);
|
||||
let target_dir = util::find_target_dir(components, &vfs)?;
|
||||
match target_dir.get(target_file) {
|
||||
Some(VNode::File(file)) => f(file),
|
||||
Some(VNode::Dir(_)) => return err::item_is_not_file(),
|
||||
None => return Err(Error::from(ErrorKind::NotFound).into()),
|
||||
}
|
||||
}
|
||||
fn with_item_mut<T>(
|
||||
fpath: &str,
|
||||
f: impl Fn(OccupiedEntry<Box<str>, VNode>) -> RuntimeResult<T>,
|
||||
) -> RuntimeResult<T> {
|
||||
let mut vfs = Self::handle().write();
|
||||
let mut current = &mut *vfs;
|
||||
// process components
|
||||
let (target, components) = util::split_target_and_components(fpath);
|
||||
for component in components {
|
||||
match current.get_mut(component) {
|
||||
Some(VNode::Dir(dir)) => {
|
||||
current = dir;
|
||||
}
|
||||
Some(VNode::File(_)) => return err::file_in_dir_path(),
|
||||
None => return err::dir_missing_in_path(),
|
||||
}
|
||||
}
|
||||
match current.entry(target.into()) {
|
||||
Entry::Occupied(item) => return f(item),
|
||||
Entry::Vacant(_) => return err::could_not_find_item(),
|
||||
}
|
||||
}
|
||||
fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> RuntimeResult<()> {
|
||||
Self::with_item_mut(fpath, |node| match node.get() {
|
||||
VNode::Dir(d) => {
|
||||
if allow_if_non_empty || d.is_empty() {
|
||||
node.remove();
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into());
|
||||
}
|
||||
VNode::File(_) => return err::file_in_dir_path(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FSInterface for VirtualFS {
|
||||
type File = VFileDescriptor;
|
||||
fn fs_rename(from: &str, to: &str) -> RuntimeResult<()> {
|
||||
// get file data
|
||||
let data = VirtualFS::with_file(from, |f| Ok(f.data.clone()))?;
|
||||
// create new file
|
||||
let file = VirtualFS::fs_fopen_or_create_rw(to)?;
|
||||
match file {
|
||||
FileOpen::Created(mut c) => {
|
||||
c.fw_write_all(&data)?;
|
||||
}
|
||||
FileOpen::Existing(mut e) => {
|
||||
e.fwext_truncate_to(0)?;
|
||||
e.fw_write_all(&data)?;
|
||||
}
|
||||
}
|
||||
// delete old file
|
||||
Self::fs_remove_file(from)
|
||||
}
|
||||
fn fs_remove_file(fpath: &str) -> RuntimeResult<()> {
|
||||
VirtualFS::with_item_mut(fpath, |e| match e.get() {
|
||||
VNode::File(_) => {
|
||||
e.remove();
|
||||
Ok(())
|
||||
}
|
||||
_ => return err::item_is_not_file(),
|
||||
})
|
||||
}
|
||||
fn fs_create_dir(fpath: &str) -> RuntimeResult<()> {
|
||||
// get vfs
|
||||
let mut vfs = VirtualFS::handle().write();
|
||||
// get root dir
|
||||
let mut current = &mut *vfs;
|
||||
// process components
|
||||
let (target, mut components) = util::split_target_and_components(fpath);
|
||||
while let Some(component) = components.next() {
|
||||
match current.get_mut(component) {
|
||||
Some(VNode::Dir(d)) => {
|
||||
current = d;
|
||||
}
|
||||
Some(VNode::File(_)) => return err::file_in_dir_path(),
|
||||
None => return err::dir_missing_in_path(),
|
||||
}
|
||||
}
|
||||
match current.entry(target.into()) {
|
||||
Entry::Occupied(_) => return Err(Error::from(ErrorKind::AlreadyExists).into()),
|
||||
Entry::Vacant(ve) => {
|
||||
ve.insert(VNode::Dir(into_dict!()));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> {
|
||||
let mut vfs = VirtualFS::handle().write();
|
||||
fn create_ahead(mut ahead: &[&str], current: &mut VDir) -> RuntimeResult<()> {
|
||||
if ahead.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let this = ahead[0];
|
||||
ahead = &ahead[1..];
|
||||
match current.get_mut(this) {
|
||||
Some(VNode::Dir(d)) => {
|
||||
if ahead.is_empty() {
|
||||
// hmm, this was the list dir that was to be created, but it already exists
|
||||
return Err(Error::from(ErrorKind::AlreadyExists).into());
|
||||
}
|
||||
return create_ahead(ahead, d);
|
||||
}
|
||||
Some(VNode::File(_)) => return err::file_in_dir_path(),
|
||||
None => {
|
||||
let _ = current.insert(this.into(), VNode::Dir(into_dict!()));
|
||||
let dir = current.get_mut(this).unwrap().as_dir_mut().unwrap();
|
||||
return create_ahead(ahead, dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
let pieces = util::split_parts(fpath);
|
||||
create_ahead(&pieces, &mut *vfs)
|
||||
}
|
||||
fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> {
|
||||
VirtualFS::delete_dir(fpath, false)
|
||||
}
|
||||
fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> {
|
||||
VirtualFS::delete_dir(fpath, true)
|
||||
}
|
||||
fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<FileOpen<Self::File>> {
|
||||
let mut vfs = VirtualFS::handle().write();
|
||||
// components
|
||||
let (target_file, components) = util::split_target_and_components(fpath);
|
||||
let target_dir = util::find_target_dir_mut(components, &mut vfs)?;
|
||||
match target_dir.entry(target_file.into()) {
|
||||
Entry::Occupied(mut oe) => match oe.get_mut() {
|
||||
VNode::File(f) => {
|
||||
f.read = true;
|
||||
f.write = true;
|
||||
Ok(FileOpen::Existing(VFileDescriptor(fpath.into())))
|
||||
}
|
||||
VNode::Dir(_) => return err::item_is_not_file(),
|
||||
},
|
||||
Entry::Vacant(v) => {
|
||||
v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
|
||||
Ok(FileOpen::Created(VFileDescriptor(fpath.into())))
|
||||
}
|
||||
}
|
||||
}
|
||||
fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
||||
let mut vfs = VirtualFS::handle().write();
|
||||
let (target_file, components) = util::split_target_and_components(fpath);
|
||||
let target_dir = util::find_target_dir_mut(components, &mut vfs)?;
|
||||
match target_dir.entry(target_file.into()) {
|
||||
Entry::Occupied(k) => {
|
||||
match k.get() {
|
||||
VNode::Dir(_) => {
|
||||
return Err(Error::new(
|
||||
ErrorKind::AlreadyExists,
|
||||
"found directory with same name where file was to be created",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
VNode::File(_) => {
|
||||
// the file already exists
|
||||
return Err(Error::new(
|
||||
ErrorKind::AlreadyExists,
|
||||
"the file already exists",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Entry::Vacant(v) => {
|
||||
// no file exists, we can create this
|
||||
v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
|
||||
Ok(VFileDescriptor(fpath.into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
||||
VirtualFS::with_file_mut(fpath, |f| {
|
||||
f.read = true;
|
||||
f.write = true;
|
||||
Ok(VFileDescriptor(fpath.into()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
vfile & descriptor impls
|
||||
(this is our `File` but a temporary, completely in-memory file)
|
||||
*/
|
||||
|
||||
pub struct VFileDescriptor(Box<str>);
|
||||
impl Drop for VFileDescriptor {
|
||||
fn drop(&mut self) {
|
||||
let _ = VirtualFS::with_file_mut(&self.0, |f| {
|
||||
f.read = false;
|
||||
f.write = false;
|
||||
f.pos = 0;
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterface for VFileDescriptor {
|
||||
type BufReader = Self;
|
||||
type BufWriter = Self;
|
||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
|
||||
Ok(self)
|
||||
}
|
||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
|
||||
Ok(self)
|
||||
}
|
||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
|
||||
Ok(r)
|
||||
}
|
||||
fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult<Self> {
|
||||
Ok(r)
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceRead for VFileDescriptor {
|
||||
fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
|
||||
VirtualFS::with_file_mut(&self.0, |file| {
|
||||
if !file.read {
|
||||
return Err(
|
||||
Error::new(ErrorKind::PermissionDenied, "Read permission denied").into(),
|
||||
);
|
||||
}
|
||||
let available_bytes = file.current().len();
|
||||
if available_bytes < buf.len() {
|
||||
return Err(Error::from(ErrorKind::UnexpectedEof).into());
|
||||
}
|
||||
buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]);
|
||||
file.pos += buf.len();
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceWrite for VFileDescriptor {
|
||||
fn fwrite(&mut self, bytes: &[u8]) -> RuntimeResult<u64> {
|
||||
VirtualFS::with_file_mut(&self.0, |file| {
|
||||
if !file.write {
|
||||
return Err(
|
||||
Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(),
|
||||
);
|
||||
}
|
||||
if file.pos + bytes.len() > file.data.len() {
|
||||
file.data.resize(file.pos + bytes.len(), 0);
|
||||
}
|
||||
file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes);
|
||||
file.pos += bytes.len();
|
||||
Ok(bytes.len() as _)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceWriteExt for VFileDescriptor {
|
||||
fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
|
||||
VirtualFS::with_file_mut(&self.0, |file| {
|
||||
if !file.write {
|
||||
return Err(
|
||||
Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(),
|
||||
);
|
||||
}
|
||||
if to as usize > file.data.len() {
|
||||
file.data.resize(to as usize, 0);
|
||||
} else {
|
||||
file.data.truncate(to as usize);
|
||||
}
|
||||
if file.pos > file.data.len() {
|
||||
file.pos = file.data.len();
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceBufWrite for VFileDescriptor {
|
||||
fn sync_write_cache(&mut self) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceExt for VFileDescriptor {
|
||||
fn fext_length(&self) -> RuntimeResult<u64> {
|
||||
VirtualFS::with_file(&self.0, |f| Ok(f.data.len() as u64))
|
||||
}
|
||||
fn fext_cursor(&mut self) -> RuntimeResult<u64> {
|
||||
VirtualFS::with_file(&self.0, |f| Ok(f.pos as u64))
|
||||
}
|
||||
fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> {
|
||||
VirtualFS::with_file_mut(&self.0, |file| {
|
||||
if by > file.data.len() as u64 {
|
||||
return Err(
|
||||
Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into(),
|
||||
);
|
||||
}
|
||||
file.pos = by as usize;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// An application level implementation of `/dev/null` with some changes
|
||||
pub struct NullFS;
|
||||
/// A handle to a file in `/dev/null` (emulated)
|
||||
pub struct NullFile;
|
||||
|
||||
impl FSInterface for NullFS {
|
||||
type File = NullFile;
|
||||
fn fs_remove_file(_: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_rename(_: &str, _: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_create_dir(_: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_create_dir_all(_: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_delete_dir(_: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_delete_dir_all(_: &str) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn fs_fopen_or_create_rw(_: &str) -> RuntimeResult<FileOpen<Self::File>> {
|
||||
Ok(FileOpen::Created(NullFile))
|
||||
}
|
||||
fn fs_fopen_rw(_: &str) -> RuntimeResult<Self::File> {
|
||||
Ok(NullFile)
|
||||
}
|
||||
fn fs_fcreate_rw(_: &str) -> RuntimeResult<Self::File> {
|
||||
Ok(NullFile)
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterface for NullFile {
|
||||
type BufReader = Self;
|
||||
type BufWriter = Self;
|
||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
|
||||
Ok(self)
|
||||
}
|
||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
|
||||
Ok(self)
|
||||
}
|
||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
|
||||
Ok(r)
|
||||
}
|
||||
fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult<Self> {
|
||||
Ok(r)
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceWrite for NullFile {
|
||||
fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult<u64> {
|
||||
Ok(buf.len() as _)
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceWriteExt for NullFile {
|
||||
fn fwext_truncate_to(&mut self, _: u64) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceRead for NullFile {
|
||||
fn fread_exact(&mut self, _: &mut [u8]) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInterfaceExt for NullFile {
|
||||
fn fext_length(&self) -> RuntimeResult<u64> {
|
||||
Ok(0)
|
||||
}
|
||||
fn fext_cursor(&mut self) -> RuntimeResult<u64> {
|
||||
Ok(0)
|
||||
}
|
||||
fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl FileInterfaceBufWrite for NullFile {
|
||||
fn sync_write_cache(&mut self) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,201 @@
|
||||
/*
|
||||
* Created on Sun Jan 07 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
use {
|
||||
crate::engine::RuntimeResult,
|
||||
std::io::{Error as IoError, ErrorKind as IoErrorKind},
|
||||
};
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
/// Result of opening a file
|
||||
/// - Created: newly created file
|
||||
/// - Existing: existing file that was reopened
|
||||
pub enum FileOpen<CF, EF = CF> {
|
||||
/// new file
|
||||
Created(CF),
|
||||
/// existing file
|
||||
Existing(EF),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<CF, EF> FileOpen<CF, EF> {
|
||||
pub fn into_existing(self) -> Option<EF> {
|
||||
match self {
|
||||
Self::Existing(e) => Some(e),
|
||||
Self::Created(_) => None,
|
||||
}
|
||||
}
|
||||
pub fn into_created(self) -> Option<CF> {
|
||||
match self {
|
||||
Self::Existing(_) => None,
|
||||
Self::Created(c) => Some(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<CF> FileOpen<CF> {
|
||||
pub fn into_inner(self) -> CF {
|
||||
match self {
|
||||
Self::Created(f) | Self::Existing(f) => f,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait FSInterface {
|
||||
// settings
|
||||
/// set to false if the file system is a special device like `/dev/null`
|
||||
const NOT_NULL: bool = true;
|
||||
// types
|
||||
/// the file type that is returned by this file system
|
||||
type File: FileInterface;
|
||||
// functions
|
||||
/// Remove a file
|
||||
fn fs_remove_file(fpath: &str) -> RuntimeResult<()>;
|
||||
/// Rename a file
|
||||
fn fs_rename(from: &str, to: &str) -> RuntimeResult<()>;
|
||||
/// Create a directory
|
||||
fn fs_create_dir(fpath: &str) -> RuntimeResult<()>;
|
||||
/// Create a directory and all corresponding path components
|
||||
fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()>;
|
||||
/// Delete a directory
|
||||
fn fs_delete_dir(fpath: &str) -> RuntimeResult<()>;
|
||||
/// Delete a directory and recursively remove all (if any) children
|
||||
fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()>;
|
||||
/// Open or create a file in R/W mode
|
||||
///
|
||||
/// This will:
|
||||
/// - Create a file if it doesn't exist
|
||||
/// - Open a file it it does exist
|
||||
fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<FileOpen<Self::File>>;
|
||||
/// Open an existing file
|
||||
fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File>;
|
||||
/// Create a new file
|
||||
fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File>;
|
||||
}
|
||||
|
||||
/// File interface definition
|
||||
pub trait FileInterface:
|
||||
FileInterfaceRead + FileInterfaceWrite + FileInterfaceWriteExt + FileInterfaceExt + Sized
|
||||
{
|
||||
/// A buffered reader implementation
|
||||
type BufReader: FileInterfaceRead + FileInterfaceExt;
|
||||
/// A buffered writer implementation
|
||||
type BufWriter: FileInterfaceBufWrite;
|
||||
/// Get a buffered reader for this file
|
||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader>;
|
||||
/// Get a buffered writer for this file
|
||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter>;
|
||||
/// Get the file back from the buffered reader
|
||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self>;
|
||||
/// Get the file back from the buffered writer
|
||||
fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult<Self>;
|
||||
}
|
||||
|
||||
pub trait FileInterfaceBufWrite: FileInterfaceWrite + FileInterfaceExt {
|
||||
fn sync_write_cache(&mut self) -> RuntimeResult<()>;
|
||||
}
|
||||
|
||||
/// Readable object
|
||||
pub trait FileInterfaceRead {
|
||||
/// Read in a block of the exact given length
|
||||
fn fread_exact_block<const N: usize>(&mut self) -> RuntimeResult<[u8; N]> {
|
||||
let mut ret = [0u8; N];
|
||||
self.fread_exact(&mut ret)?;
|
||||
Ok(ret)
|
||||
}
|
||||
/// Read in `n` bytes to fill the given buffer
|
||||
fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()>;
|
||||
}
|
||||
|
||||
/// Writable object
|
||||
pub trait FileInterfaceWrite {
|
||||
/// Attempt to write the buffer into this object, returning the number of bytes that were
|
||||
/// written. It is **NOT GUARANTEED THAT ALL DATA WILL BE WRITTEN**
|
||||
fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult<u64>;
|
||||
/// Attempt to write the entire buffer into this object, returning the number of bytes written
|
||||
///
|
||||
/// It is guaranteed that if the [`Result`] returned is [`Ok(())`], then the entire buffer was
|
||||
/// written to disk.
|
||||
fn fwrite_all_count(&mut self, buf: &[u8]) -> (u64, RuntimeResult<()>) {
|
||||
let len = buf.len() as u64;
|
||||
let mut written = 0;
|
||||
while written != len {
|
||||
match self.fwrite(buf) {
|
||||
Ok(0) => {
|
||||
return (
|
||||
written,
|
||||
Err(IoError::new(
|
||||
IoErrorKind::WriteZero,
|
||||
format!("could only write {} of {} bytes", written, buf.len()),
|
||||
)
|
||||
.into()),
|
||||
)
|
||||
}
|
||||
Ok(n) => written += n,
|
||||
Err(e) => return (written, Err(e)),
|
||||
}
|
||||
}
|
||||
(written, Ok(()))
|
||||
}
|
||||
/// Attempt to write the entire buffer into this object
|
||||
///
|
||||
/// If this return [`Ok(())`] then it is guaranteed that all bytes have been written
|
||||
fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()> {
|
||||
self.fwrite_all_count(buf).1
|
||||
}
|
||||
}
|
||||
|
||||
/// Advanced write traits
|
||||
pub trait FileInterfaceWriteExt {
|
||||
/// Sync data and metadata for this file
|
||||
fn fwext_sync_all(&mut self) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Sync data for this file
|
||||
fn fwext_sync_data(&mut self) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Sync meta for this file
|
||||
fn fwext_sync_meta(&mut self) -> RuntimeResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
/// Truncate the size of the file to the given size
|
||||
///
|
||||
/// - If `to` > actual file length: the file is zero padded to fill `to - len`
|
||||
/// - If `to` < actual file length: the file is trimmed to the size `to`
|
||||
fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()>;
|
||||
}
|
||||
|
||||
/// Advanced file access
|
||||
pub trait FileInterfaceExt {
|
||||
/// Returns the length of the file
|
||||
fn fext_length(&self) -> RuntimeResult<u64>;
|
||||
/// Returns the current cursor position of the file
|
||||
fn fext_cursor(&mut self) -> RuntimeResult<u64>;
|
||||
/// Seek by `from` bytes from the start of the file
|
||||
fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()>;
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Created on Sun Jan 07 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
//! # FS abstractions
|
||||
//!
|
||||
//! This module defines abstractions over file systems (whether physical or virtual) and provides
|
||||
//! traits that provide an unified API for all file systems irrespective of their base impl
|
||||
//!
|
||||
|
||||
pub mod fs_imp;
|
||||
#[cfg(test)]
|
||||
pub mod fs_test;
|
||||
pub mod fs_traits;
|
||||
@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Created on Tue Jan 09 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
pub mod checksum;
|
||||
pub mod interface;
|
||||
pub mod sdss;
|
||||
pub mod static_meta;
|
||||
pub mod versions;
|
||||
@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Created on Fri Jan 12 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
mod spec;
|
||||
pub use spec::{FileSpecV1, HeaderV1, HeaderV1Enumeration, HeaderV1Spec, SimpleFileSpecV1};
|
||||
@ -0,0 +1,485 @@
|
||||
/*
|
||||
* Created on Wed Jan 10 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
/*!
|
||||
# SDSS spec
|
||||
|
||||
This module provides traits and types to deal with the SDSS spec, especially headers.
|
||||
|
||||
The static SDSS header block has a special segment that defines the header version which is static and will
|
||||
never change across any versions. While the same isn't warranted for the rest of the header, it's exceedingly
|
||||
unlikely that we'll ever change the static block.
|
||||
|
||||
The only header that we currently use is [`HeaderV1`].
|
||||
*/
|
||||
|
||||
use {
|
||||
super::super::{
|
||||
interface::fs_traits::{FileInterfaceRead, FileInterfaceWrite},
|
||||
static_meta::{HostArch, HostEndian, HostOS, HostPointerWidth, SDSS_MAGIC_8B},
|
||||
versions::{self, DriverVersion, FileSpecifierVersion, HeaderVersion, ServerVersion},
|
||||
},
|
||||
crate::{
|
||||
engine::{error::StorageError, mem::memcpy, RuntimeResult},
|
||||
util::os,
|
||||
},
|
||||
std::{
|
||||
mem::{transmute, ManuallyDrop},
|
||||
ops::Range,
|
||||
},
|
||||
};
|
||||
|
||||
/*
|
||||
header utils
|
||||
*/
|
||||
|
||||
pub trait HeaderV1Enumeration {
|
||||
/// the maximum value of this enumeration
|
||||
const MAX: u8;
|
||||
/// Create a new enumeration, given that the maximum is validated
|
||||
unsafe fn new(x: u8) -> Self;
|
||||
/// Return the 1B repr of the enumeration
|
||||
fn repr_u8(&self) -> u8;
|
||||
}
|
||||
|
||||
/// A trait that enables customizing the SDSS header for a specific version tuple
|
||||
pub trait HeaderV1Spec {
|
||||
// types
|
||||
/// The file class type
|
||||
type FileClass: HeaderV1Enumeration + Copy + PartialEq;
|
||||
/// The file specifier type
|
||||
type FileSpecifier: HeaderV1Enumeration + Copy + PartialEq;
|
||||
// constants
|
||||
/// The server version to use during encode
|
||||
///
|
||||
/// NB: This is NOT the compatible version but rather the current version
|
||||
const CURRENT_SERVER_VERSION: ServerVersion;
|
||||
/// The driver version to use during encode
|
||||
///
|
||||
/// NB: This is NOT the compatible version but rather the current version
|
||||
const CURRENT_DRIVER_VERSION: DriverVersion;
|
||||
/// The file class to use and verify at encode/decode time
|
||||
/// check server version compatibility is valid at decode time
|
||||
fn check_if_server_version_compatible(v: ServerVersion) -> bool {
|
||||
v == Self::CURRENT_SERVER_VERSION
|
||||
}
|
||||
/// check driver version compatibility is valid at decode time
|
||||
fn check_if_driver_version_compatible(v: DriverVersion) -> bool {
|
||||
v == Self::CURRENT_DRIVER_VERSION
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Compact SDSS Header v1
|
||||
---
|
||||
- 1: Magic block (16B): magic + header version
|
||||
- 2: Static block (40B):
|
||||
- 2.1: Genesis static record (24B)
|
||||
- 2.1.1: Software information (16B)
|
||||
- Server version (8B)
|
||||
- Driver version (8B)
|
||||
- 2.1.2: Host information (4B):
|
||||
- OS (1B)
|
||||
- Arch (1B)
|
||||
- Pointer width (1B)
|
||||
- Endian (1B)
|
||||
- 2.1.3: File information (4B):
|
||||
- File class (1B)
|
||||
- File specifier (1B)
|
||||
- File specifier version (2B)
|
||||
- 2.2: Genesis runtime record (16B)
|
||||
- Host epoch (16B)
|
||||
- 3: Padding block (8B)
|
||||
*/
|
||||
|
||||
#[repr(align(8))]
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct HeaderV1<H: HeaderV1Spec> {
|
||||
// 1 magic block
|
||||
magic_header_version: HeaderVersion,
|
||||
// 2.1.1
|
||||
genesis_static_sw_server_version: ServerVersion,
|
||||
genesis_static_sw_driver_version: DriverVersion,
|
||||
// 2.1.2
|
||||
genesis_static_host_os: HostOS,
|
||||
genesis_static_host_arch: HostArch,
|
||||
genesis_static_host_ptr_width: HostPointerWidth,
|
||||
genesis_static_host_endian: HostEndian,
|
||||
// 2.1.3
|
||||
genesis_static_file_class: H::FileClass,
|
||||
genesis_static_file_specifier: H::FileSpecifier,
|
||||
genesis_static_file_specifier_version: FileSpecifierVersion,
|
||||
// 2.2
|
||||
genesis_runtime_epoch_time: u128,
|
||||
// 3
|
||||
genesis_padding_block: [u8; 8],
|
||||
}
|
||||
|
||||
impl<H: HeaderV1Spec> HeaderV1<H> {
|
||||
const SEG1_MAGIC: Range<usize> = 0..8;
|
||||
const SEG1_HEADER_VERSION: Range<usize> = 8..16;
|
||||
const SEG2_REC1_SERVER_VERSION: Range<usize> = 16..24;
|
||||
const SEG2_REC1_DRIVER_VERSION: Range<usize> = 24..32;
|
||||
const SEG2_REC1_HOST_OS: usize = 32;
|
||||
const SEG2_REC1_HOST_ARCH: usize = 33;
|
||||
const SEG2_REC1_HOST_PTR_WIDTH: usize = 34;
|
||||
const SEG2_REC1_HOST_ENDIAN: usize = 35;
|
||||
const SEG2_REC1_FILE_CLASS: usize = 36;
|
||||
const SEG2_REC1_FILE_SPECIFIER: usize = 37;
|
||||
const SEG2_REC1_FILE_SPECIFIER_VERSION: Range<usize> = 38..40;
|
||||
const SEG2_REC2_RUNTIME_EPOCH_TIME: Range<usize> = 40..56;
|
||||
const SEG3_PADDING_BLK: Range<usize> = 56..64;
|
||||
pub const SIZE: usize = 64;
|
||||
#[inline(always)]
|
||||
fn _new_auto(
|
||||
file_class: H::FileClass,
|
||||
file_specifier: H::FileSpecifier,
|
||||
file_specifier_version: FileSpecifierVersion,
|
||||
epoch_time: u128,
|
||||
genesis_padding_block: [u8; 8],
|
||||
) -> Self {
|
||||
Self::_new(
|
||||
versions::HEADER_V1,
|
||||
H::CURRENT_SERVER_VERSION,
|
||||
H::CURRENT_DRIVER_VERSION,
|
||||
HostOS::new(),
|
||||
HostArch::new(),
|
||||
HostPointerWidth::new(),
|
||||
HostEndian::new(),
|
||||
file_class,
|
||||
file_specifier,
|
||||
file_specifier_version,
|
||||
epoch_time,
|
||||
genesis_padding_block,
|
||||
)
|
||||
}
|
||||
#[inline(always)]
|
||||
fn _new(
|
||||
magic_header_version: HeaderVersion,
|
||||
genesis_static_sw_server_version: ServerVersion,
|
||||
genesis_static_sw_driver_version: DriverVersion,
|
||||
genesis_static_host_os: HostOS,
|
||||
genesis_static_host_arch: HostArch,
|
||||
genesis_static_host_ptr_width: HostPointerWidth,
|
||||
genesis_static_host_endian: HostEndian,
|
||||
genesis_static_file_class: H::FileClass,
|
||||
genesis_static_file_specifier: H::FileSpecifier,
|
||||
genesis_static_file_specifier_version: FileSpecifierVersion,
|
||||
genesis_runtime_epoch_time: u128,
|
||||
genesis_padding_block: [u8; 8],
|
||||
) -> Self {
|
||||
Self {
|
||||
magic_header_version,
|
||||
genesis_static_sw_server_version,
|
||||
genesis_static_sw_driver_version,
|
||||
genesis_static_host_os,
|
||||
genesis_static_host_arch,
|
||||
genesis_static_host_ptr_width,
|
||||
genesis_static_host_endian,
|
||||
genesis_static_file_class,
|
||||
genesis_static_file_specifier,
|
||||
genesis_static_file_specifier_version,
|
||||
genesis_runtime_epoch_time,
|
||||
genesis_padding_block,
|
||||
}
|
||||
}
|
||||
fn _encode_auto_raw(
|
||||
file_class: H::FileClass,
|
||||
file_specifier: H::FileSpecifier,
|
||||
file_specifier_version: FileSpecifierVersion,
|
||||
epoch_time: u128,
|
||||
padding_block: [u8; 8],
|
||||
) -> [u8; 64] {
|
||||
let mut ret = [0; 64];
|
||||
// 1. mgblk
|
||||
ret[Self::SEG1_MAGIC].copy_from_slice(&SDSS_MAGIC_8B.to_le_bytes());
|
||||
ret[Self::SEG1_HEADER_VERSION]
|
||||
.copy_from_slice(&versions::v1::V1_HEADER_VERSION.little_endian_u64());
|
||||
// 2.1.1
|
||||
ret[Self::SEG2_REC1_SERVER_VERSION]
|
||||
.copy_from_slice(&H::CURRENT_SERVER_VERSION.little_endian());
|
||||
ret[Self::SEG2_REC1_DRIVER_VERSION]
|
||||
.copy_from_slice(&H::CURRENT_DRIVER_VERSION.little_endian());
|
||||
// 2.1.2
|
||||
ret[Self::SEG2_REC1_HOST_OS] = HostOS::new().value_u8();
|
||||
ret[Self::SEG2_REC1_HOST_ARCH] = HostArch::new().value_u8();
|
||||
ret[Self::SEG2_REC1_HOST_PTR_WIDTH] = HostPointerWidth::new().value_u8();
|
||||
ret[Self::SEG2_REC1_HOST_ENDIAN] = HostEndian::new().value_u8();
|
||||
// 2.1.3
|
||||
ret[Self::SEG2_REC1_FILE_CLASS] = file_class.repr_u8();
|
||||
ret[Self::SEG2_REC1_FILE_SPECIFIER] = file_specifier.repr_u8();
|
||||
ret[Self::SEG2_REC1_FILE_SPECIFIER_VERSION]
|
||||
.copy_from_slice(&file_specifier_version.little_endian());
|
||||
// 2.2
|
||||
ret[Self::SEG2_REC2_RUNTIME_EPOCH_TIME].copy_from_slice(&epoch_time.to_le_bytes());
|
||||
// 3
|
||||
ret[Self::SEG3_PADDING_BLK].copy_from_slice(&padding_block);
|
||||
ret
|
||||
}
|
||||
fn encode_return(
|
||||
file_class: H::FileClass,
|
||||
file_specifier: H::FileSpecifier,
|
||||
file_specifier_version: FileSpecifierVersion,
|
||||
) -> (Self, [u8; 64]) {
|
||||
let epoch_time = os::get_epoch_time();
|
||||
let encoded = Self::_encode_auto_raw(
|
||||
file_class,
|
||||
file_specifier,
|
||||
file_specifier_version,
|
||||
epoch_time,
|
||||
[0; 8],
|
||||
);
|
||||
let me = Self::_new_auto(
|
||||
file_class,
|
||||
file_specifier,
|
||||
file_specifier_version,
|
||||
epoch_time,
|
||||
[0; 8],
|
||||
);
|
||||
(me, encoded)
|
||||
}
|
||||
/// Decode and validate the full header block (validate ONLY; you must verify yourself)
|
||||
///
|
||||
/// Notes:
|
||||
/// - Time might be inconsistent; verify
|
||||
/// - Compatibility requires additional intervention
|
||||
/// - If padding block was not zeroed, handle
|
||||
/// - No file metadata is verified. Check!
|
||||
///
|
||||
pub fn decode(block: [u8; 64]) -> Result<Self, StorageError> {
|
||||
var!(let raw_magic, raw_header_version, raw_server_version, raw_driver_version, raw_host_os, raw_host_arch,
|
||||
raw_host_ptr_width, raw_host_endian, raw_file_class, raw_file_specifier, raw_file_specifier_version,
|
||||
raw_runtime_epoch_time, raw_paddding_block,
|
||||
);
|
||||
macro_rules! u64 {
|
||||
($pos:expr) => {
|
||||
u64::from_le_bytes(memcpy(&block[$pos]))
|
||||
};
|
||||
}
|
||||
unsafe {
|
||||
// UNSAFE(@ohsayan): all segments are correctly accessed (aligned to u8)
|
||||
raw_magic = u64!(Self::SEG1_MAGIC);
|
||||
raw_header_version = HeaderVersion::__new(u64!(Self::SEG1_HEADER_VERSION));
|
||||
raw_server_version = ServerVersion::__new(u64!(Self::SEG2_REC1_SERVER_VERSION));
|
||||
raw_driver_version = DriverVersion::__new(u64!(Self::SEG2_REC1_DRIVER_VERSION));
|
||||
raw_host_os = block[Self::SEG2_REC1_HOST_OS];
|
||||
raw_host_arch = block[Self::SEG2_REC1_HOST_ARCH];
|
||||
raw_host_ptr_width = block[Self::SEG2_REC1_HOST_PTR_WIDTH];
|
||||
raw_host_endian = block[Self::SEG2_REC1_HOST_ENDIAN];
|
||||
raw_file_class = block[Self::SEG2_REC1_FILE_CLASS];
|
||||
raw_file_specifier = block[Self::SEG2_REC1_FILE_SPECIFIER];
|
||||
raw_file_specifier_version = FileSpecifierVersion::__new(u16::from_le_bytes(memcpy(
|
||||
&block[Self::SEG2_REC1_FILE_SPECIFIER_VERSION],
|
||||
)));
|
||||
raw_runtime_epoch_time =
|
||||
u128::from_le_bytes(memcpy(&block[Self::SEG2_REC2_RUNTIME_EPOCH_TIME]));
|
||||
raw_paddding_block = memcpy::<8>(&block[Self::SEG3_PADDING_BLK]);
|
||||
}
|
||||
let okay_header_version = raw_header_version == versions::HEADER_V1;
|
||||
let okay_server_version = H::check_if_server_version_compatible(raw_server_version);
|
||||
let okay_driver_version = H::check_if_driver_version_compatible(raw_driver_version);
|
||||
let okay = okay!(
|
||||
// 1.1 mgblk
|
||||
raw_magic == SDSS_MAGIC_8B,
|
||||
okay_header_version,
|
||||
// 2.1.1
|
||||
okay_server_version,
|
||||
okay_driver_version,
|
||||
// 2.1.2
|
||||
raw_host_os <= HostOS::MAX,
|
||||
raw_host_arch <= HostArch::MAX,
|
||||
raw_host_ptr_width <= HostPointerWidth::MAX,
|
||||
raw_host_endian <= HostEndian::MAX,
|
||||
// 2.1.3
|
||||
raw_file_class <= H::FileClass::MAX,
|
||||
raw_file_specifier <= H::FileSpecifier::MAX,
|
||||
);
|
||||
if okay {
|
||||
Ok(unsafe {
|
||||
// UNSAFE(@ohsayan): the block ranges are very well defined
|
||||
Self::_new(
|
||||
// 1.1
|
||||
raw_header_version,
|
||||
// 2.1.1
|
||||
raw_server_version,
|
||||
raw_driver_version,
|
||||
// 2.1.2
|
||||
transmute(raw_host_os),
|
||||
transmute(raw_host_arch),
|
||||
transmute(raw_host_ptr_width),
|
||||
transmute(raw_host_endian),
|
||||
// 2.1.3
|
||||
H::FileClass::new(raw_file_class),
|
||||
H::FileSpecifier::new(raw_file_specifier),
|
||||
raw_file_specifier_version,
|
||||
// 2.2
|
||||
raw_runtime_epoch_time,
|
||||
// 3
|
||||
raw_paddding_block,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
let version_okay = okay_header_version & okay_server_version & okay_driver_version;
|
||||
let md = ManuallyDrop::new([
|
||||
StorageError::HeaderDecodeCorruptedHeader,
|
||||
StorageError::HeaderDecodeVersionMismatch,
|
||||
]);
|
||||
Err(unsafe {
|
||||
// UNSAFE(@ohsayan): while not needed, md for drop safety + correct index
|
||||
md.as_ptr().add(!version_okay as usize).read().into()
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
impl<H: HeaderV1Spec> HeaderV1<H> {
|
||||
pub fn header_version(&self) -> HeaderVersion {
|
||||
self.magic_header_version
|
||||
}
|
||||
pub fn server_version(&self) -> ServerVersion {
|
||||
self.genesis_static_sw_server_version
|
||||
}
|
||||
pub fn driver_version(&self) -> DriverVersion {
|
||||
self.genesis_static_sw_driver_version
|
||||
}
|
||||
pub fn host_os(&self) -> HostOS {
|
||||
self.genesis_static_host_os
|
||||
}
|
||||
pub fn host_arch(&self) -> HostArch {
|
||||
self.genesis_static_host_arch
|
||||
}
|
||||
pub fn host_ptr_width(&self) -> HostPointerWidth {
|
||||
self.genesis_static_host_ptr_width
|
||||
}
|
||||
pub fn host_endian(&self) -> HostEndian {
|
||||
self.genesis_static_host_endian
|
||||
}
|
||||
pub fn file_class(&self) -> H::FileClass {
|
||||
self.genesis_static_file_class
|
||||
}
|
||||
pub fn file_specifier(&self) -> H::FileSpecifier {
|
||||
self.genesis_static_file_specifier
|
||||
}
|
||||
pub fn file_specifier_version(&self) -> FileSpecifierVersion {
|
||||
self.genesis_static_file_specifier_version
|
||||
}
|
||||
pub fn epoch_time(&self) -> u128 {
|
||||
self.genesis_runtime_epoch_time
|
||||
}
|
||||
pub fn padding_block(&self) -> [u8; 8] {
|
||||
self.genesis_padding_block
|
||||
}
|
||||
}
|
||||
|
||||
pub trait FileSpecV1 {
|
||||
type Metadata;
|
||||
/// the header type
|
||||
type HeaderSpec: HeaderV1Spec;
|
||||
/// the args need to validate the metadata (for example, additional context)
|
||||
type EncodeArgs;
|
||||
type DecodeArgs;
|
||||
/// validate the metadata
|
||||
fn validate_metadata(
|
||||
md: HeaderV1<Self::HeaderSpec>,
|
||||
v_args: Self::DecodeArgs,
|
||||
) -> RuntimeResult<Self::Metadata>;
|
||||
/// read and validate metadata (only override if you need to)
|
||||
fn read_metadata(
|
||||
f: &mut impl FileInterfaceRead,
|
||||
v_args: Self::DecodeArgs,
|
||||
) -> RuntimeResult<Self::Metadata> {
|
||||
let md = HeaderV1::decode(f.fread_exact_block()?)?;
|
||||
Self::validate_metadata(md, v_args)
|
||||
}
|
||||
/// write metadata
|
||||
fn write_metadata(
|
||||
f: &mut impl FileInterfaceWrite,
|
||||
args: Self::EncodeArgs,
|
||||
) -> RuntimeResult<Self::Metadata>;
|
||||
}
|
||||
|
||||
/// # Simple SDSS file specification (v1)
|
||||
///
|
||||
/// ## Decode and verify
|
||||
/// A simple SDSS file specification that checks if:
|
||||
/// - the file class,
|
||||
/// - file specifier and
|
||||
/// - file specifier revision
|
||||
///
|
||||
/// match
|
||||
///
|
||||
/// ## Version Compatibility
|
||||
///
|
||||
/// Also, the [`HeaderV1Spec`] is supposed to address compatibility across server and driver versions
|
||||
pub trait SimpleFileSpecV1 {
|
||||
type HeaderSpec: HeaderV1Spec;
|
||||
const FILE_CLASS: <Self::HeaderSpec as HeaderV1Spec>::FileClass;
|
||||
const FILE_SPECIFIER: <Self::HeaderSpec as HeaderV1Spec>::FileSpecifier;
|
||||
const FILE_SPECFIER_VERSION: FileSpecifierVersion;
|
||||
fn check_if_file_specifier_revision_is_compatible(
|
||||
v: FileSpecifierVersion,
|
||||
) -> RuntimeResult<()> {
|
||||
if v == Self::FILE_SPECFIER_VERSION {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StorageError::HeaderDecodeVersionMismatch.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Sfs: SimpleFileSpecV1> FileSpecV1 for Sfs {
|
||||
type Metadata = HeaderV1<Self::HeaderSpec>;
|
||||
type HeaderSpec = <Self as SimpleFileSpecV1>::HeaderSpec;
|
||||
type DecodeArgs = ();
|
||||
type EncodeArgs = ();
|
||||
fn validate_metadata(
|
||||
md: HeaderV1<Self::HeaderSpec>,
|
||||
_: Self::DecodeArgs,
|
||||
) -> RuntimeResult<Self::Metadata> {
|
||||
let okay = okay!(
|
||||
md.file_class() == Self::FILE_CLASS,
|
||||
md.file_specifier() == Self::FILE_SPECIFIER,
|
||||
);
|
||||
Self::check_if_file_specifier_revision_is_compatible(md.file_specifier_version())?;
|
||||
if okay {
|
||||
Ok(md)
|
||||
} else {
|
||||
Err(StorageError::HeaderDecodeVersionMismatch.into())
|
||||
}
|
||||
}
|
||||
fn write_metadata(
|
||||
f: &mut impl FileInterfaceWrite,
|
||||
_: Self::EncodeArgs,
|
||||
) -> RuntimeResult<Self::Metadata> {
|
||||
let (md, block) = HeaderV1::<Self::HeaderSpec>::encode_return(
|
||||
Self::FILE_CLASS,
|
||||
Self::FILE_SPECIFIER,
|
||||
Self::FILE_SPECFIER_VERSION,
|
||||
);
|
||||
f.fw_write_all(&block).map(|_| md)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,160 @@
|
||||
/*
|
||||
* Created on Tue Jan 09 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
//! # Static metadata
|
||||
//!
|
||||
//! Compile-time metadata used by storage engine implementations
|
||||
//!
|
||||
|
||||
/// The 8B SDSS magic block
|
||||
pub const SDSS_MAGIC_8B: u64 = 0x4F48534159414E21;
|
||||
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
/// Host architecture enumeration for common platforms
|
||||
pub enum HostArch {
|
||||
X86 = 0,
|
||||
X86_64 = 1,
|
||||
ARM = 2,
|
||||
ARM64 = 3,
|
||||
MIPS = 4,
|
||||
PowerPC = 5,
|
||||
}
|
||||
|
||||
impl HostArch {
|
||||
pub const fn new() -> Self {
|
||||
if cfg!(target_arch = "x86") {
|
||||
HostArch::X86
|
||||
} else if cfg!(target_arch = "x86_64") {
|
||||
HostArch::X86_64
|
||||
} else if cfg!(target_arch = "arm") {
|
||||
HostArch::ARM
|
||||
} else if cfg!(target_arch = "aarch64") {
|
||||
HostArch::ARM64
|
||||
} else if cfg!(target_arch = "mips") {
|
||||
HostArch::MIPS
|
||||
} else if cfg!(target_arch = "powerpc") {
|
||||
HostArch::PowerPC
|
||||
} else {
|
||||
panic!("Unsupported target architecture")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
/// Host OS enumeration for common operating systems
|
||||
pub enum HostOS {
|
||||
// T1
|
||||
Linux = 0,
|
||||
Windows = 1,
|
||||
MacOS = 2,
|
||||
// T2
|
||||
Android = 3,
|
||||
AppleiOS = 4,
|
||||
FreeBSD = 5,
|
||||
OpenBSD = 6,
|
||||
NetBSD = 7,
|
||||
WASI = 8,
|
||||
Emscripten = 9,
|
||||
// T3
|
||||
Solaris = 10,
|
||||
Fuchsia = 11,
|
||||
Redox = 12,
|
||||
DragonFly = 13,
|
||||
}
|
||||
|
||||
impl HostOS {
|
||||
pub const fn new() -> Self {
|
||||
if cfg!(target_os = "linux") {
|
||||
HostOS::Linux
|
||||
} else if cfg!(target_os = "windows") {
|
||||
HostOS::Windows
|
||||
} else if cfg!(target_os = "macos") {
|
||||
HostOS::MacOS
|
||||
} else if cfg!(target_os = "android") {
|
||||
HostOS::Android
|
||||
} else if cfg!(target_os = "ios") {
|
||||
HostOS::AppleiOS
|
||||
} else if cfg!(target_os = "freebsd") {
|
||||
HostOS::FreeBSD
|
||||
} else if cfg!(target_os = "openbsd") {
|
||||
HostOS::OpenBSD
|
||||
} else if cfg!(target_os = "netbsd") {
|
||||
HostOS::NetBSD
|
||||
} else if cfg!(target_os = "dragonfly") {
|
||||
HostOS::DragonFly
|
||||
} else if cfg!(target_os = "redox") {
|
||||
HostOS::Redox
|
||||
} else if cfg!(target_os = "fuchsia") {
|
||||
HostOS::Fuchsia
|
||||
} else if cfg!(target_os = "solaris") {
|
||||
HostOS::Solaris
|
||||
} else if cfg!(target_os = "emscripten") {
|
||||
HostOS::Emscripten
|
||||
} else if cfg!(target_os = "wasi") {
|
||||
HostOS::WASI
|
||||
} else {
|
||||
panic!("unknown os")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
/// Host endian enumeration
|
||||
pub enum HostEndian {
|
||||
Big = 0,
|
||||
Little = 1,
|
||||
}
|
||||
|
||||
impl HostEndian {
|
||||
pub const fn new() -> Self {
|
||||
if cfg!(target_endian = "little") {
|
||||
Self::Little
|
||||
} else {
|
||||
Self::Big
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
#[repr(u8)]
|
||||
/// Host pointer width enumeration
|
||||
pub enum HostPointerWidth {
|
||||
P32 = 0,
|
||||
P64 = 1,
|
||||
}
|
||||
|
||||
impl HostPointerWidth {
|
||||
pub const fn new() -> Self {
|
||||
match sizeof!(usize) {
|
||||
4 => Self::P32,
|
||||
8 => Self::P64,
|
||||
_ => panic!("unknown pointer width"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Created on Mon May 15 2023
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
//! # Versioning
|
||||
//!
|
||||
//! Storage engine versioning utility
|
||||
//!
|
||||
|
||||
pub mod server_version;
|
||||
|
||||
pub const HEADER_V1: HeaderVersion = HeaderVersion(0);
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
/// The header version
|
||||
///
|
||||
/// The header version is part of the static record and *barely* changes (almost like once in a light year)
|
||||
pub struct HeaderVersion(u64);
|
||||
|
||||
impl HeaderVersion {
|
||||
pub const fn __new(v: u64) -> Self {
|
||||
Self(v)
|
||||
}
|
||||
pub const fn little_endian_u64(&self) -> [u8; 8] {
|
||||
self.0.to_le_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
/// The server version (based on tag index)
|
||||
pub struct ServerVersion(u64);
|
||||
|
||||
impl ServerVersion {
|
||||
pub const fn __new(v: u64) -> Self {
|
||||
Self(v)
|
||||
}
|
||||
pub const fn little_endian(&self) -> [u8; 8] {
|
||||
self.0.to_le_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
/// The driver version
|
||||
pub struct DriverVersion(u64);
|
||||
|
||||
impl DriverVersion {
|
||||
pub const fn __new(v: u64) -> Self {
|
||||
Self(v)
|
||||
}
|
||||
pub const fn little_endian(&self) -> [u8; 8] {
|
||||
self.0.to_le_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
/// The file specifier version
|
||||
pub struct FileSpecifierVersion(u16);
|
||||
|
||||
impl FileSpecifierVersion {
|
||||
pub const fn __new(v: u16) -> Self {
|
||||
Self(v)
|
||||
}
|
||||
pub const fn little_endian(&self) -> [u8; 2] {
|
||||
self.0.to_le_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
pub mod v1 {
|
||||
//! The first SDSS based storage engine implementation.
|
||||
//! Target tag: 0.8.0 {beta.1, beta.2, beta.3}
|
||||
use super::{DriverVersion, HeaderVersion, ServerVersion};
|
||||
|
||||
/// The SDSS header version UID
|
||||
pub const V1_HEADER_VERSION: HeaderVersion = HeaderVersion(0);
|
||||
/// The server version UID
|
||||
pub const V1_SERVER_VERSION: ServerVersion =
|
||||
ServerVersion(super::server_version::fetch_id("v0.8.0") as _);
|
||||
/// The driver version UID
|
||||
pub const V1_DRIVER_VERSION: DriverVersion = DriverVersion(0);
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub mod v2 {
|
||||
//! The second SDSS based storage implementation
|
||||
//!
|
||||
//! Target tag: 0.8.0 (GA)
|
||||
//!
|
||||
//! Same tags as [`super::v1`] but different [`DriverVersion`]
|
||||
use super::{DriverVersion, HeaderVersion, ServerVersion};
|
||||
pub const V2_HEADER_VERSION: HeaderVersion = super::v1::V1_HEADER_VERSION;
|
||||
pub const V2_SERVER_VERSION: ServerVersion = super::v1::V1_SERVER_VERSION;
|
||||
pub const V2_DRIVER_VERSION: DriverVersion = DriverVersion(1);
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Created on Wed May 17 2023
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
const VERSION_TAGS: [&str; 52] = [
|
||||
"v0.1.0",
|
||||
"v0.2.0",
|
||||
"v0.3.0",
|
||||
"v0.3.1",
|
||||
"v0.3.2",
|
||||
"v0.4.0-alpha.1",
|
||||
"v0.4.0-alpha.2",
|
||||
"v0.4.0",
|
||||
"v0.4.1-alpha.1",
|
||||
"v0.4.1",
|
||||
"v0.4.2-alpha.1",
|
||||
"v0.4.2",
|
||||
"v0.4.3-alpha.1",
|
||||
"v0.4.3",
|
||||
"v0.4.4",
|
||||
"v0.4.5-alpha.1",
|
||||
"v0.4.5-alpha.2",
|
||||
"v0.4.5",
|
||||
"v0.5.0-alpha.1",
|
||||
"v0.5.0-alpha.2",
|
||||
"v0.5.0",
|
||||
"v0.5.1-alpha.1",
|
||||
"v0.5.1",
|
||||
"v0.5.2",
|
||||
"v0.5.3",
|
||||
"v0.6.0",
|
||||
"v0.6.1",
|
||||
"v0.6.2-testrelease.1",
|
||||
"v0.6.2",
|
||||
"v0.6.3-alpha.1",
|
||||
"v0.6.3",
|
||||
"v0.6.4-alpha.1",
|
||||
"v0.6.4",
|
||||
"v0.7.0-RC.1",
|
||||
"v0.7.0-alpha.1",
|
||||
"v0.7.0-alpha.2",
|
||||
"v0.7.0-beta.1",
|
||||
"v0.7.0",
|
||||
"v0.7.1-alpha.1",
|
||||
"v0.7.1",
|
||||
"v0.7.2-alpha.1",
|
||||
"v0.7.2",
|
||||
"v0.7.3-alpha.1",
|
||||
"v0.7.3-alpha.2",
|
||||
"v0.7.3-alpha.3",
|
||||
"v0.7.3",
|
||||
"v0.7.4",
|
||||
"v0.7.5",
|
||||
"v0.7.6",
|
||||
"v0.7.7",
|
||||
"v0.8.0-alpha.1",
|
||||
"v0.8.0",
|
||||
];
|
||||
const VERSION_TAGS_LEN: usize = VERSION_TAGS.len();
|
||||
pub const fn fetch_id(id: &str) -> usize {
|
||||
// this is ct, so a O(n) doesn't matter
|
||||
let mut i = 0;
|
||||
while i < VERSION_TAGS_LEN {
|
||||
let bytes = VERSION_TAGS[i].as_bytes();
|
||||
let given = id.as_bytes();
|
||||
let mut j = 0;
|
||||
let mut eq = true;
|
||||
while (j < bytes.len()) & (bytes.len() == given.len()) {
|
||||
if bytes[i] != given[i] {
|
||||
eq = false;
|
||||
break;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
if eq {
|
||||
return i;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
panic!("version not found")
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Created on Sun Jan 07 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
mod spec;
|
||||
@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Created on Thu Jan 11 2024
|
||||
*
|
||||
* This file is a part of Skytable
|
||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||
* vision to provide flexibility in data modelling without compromising
|
||||
* on performance, queryability or scalability.
|
||||
*
|
||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
use {
|
||||
crate::engine::storage::common::{
|
||||
sdss::{self, HeaderV1Spec},
|
||||
versions::{self, DriverVersion, FileSpecifierVersion, ServerVersion},
|
||||
},
|
||||
std::mem::transmute,
|
||||
};
|
||||
|
||||
/// The file scope
|
||||
#[repr(u8)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
pub enum FileClass {
|
||||
EventLog = 0,
|
||||
Batch = 1,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
|
||||
#[repr(u8)]
|
||||
pub enum FileSpecifier {
|
||||
GlobalNS = 0,
|
||||
ModelData = 1,
|
||||
}
|
||||
|
||||
impl sdss::HeaderV1Enumeration for FileClass {
|
||||
const MAX: u8 = FileClass::MAX;
|
||||
unsafe fn new(x: u8) -> Self {
|
||||
transmute(x)
|
||||
}
|
||||
fn repr_u8(&self) -> u8 {
|
||||
self.value_u8()
|
||||
}
|
||||
}
|
||||
|
||||
impl sdss::HeaderV1Enumeration for FileSpecifier {
|
||||
const MAX: u8 = FileSpecifier::MAX;
|
||||
unsafe fn new(x: u8) -> Self {
|
||||
transmute(x)
|
||||
}
|
||||
fn repr_u8(&self) -> u8 {
|
||||
self.value_u8()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HeaderImplV2;
|
||||
impl HeaderV1Spec for HeaderImplV2 {
|
||||
type FileClass = FileClass;
|
||||
type FileSpecifier = FileSpecifier;
|
||||
const CURRENT_SERVER_VERSION: ServerVersion = versions::v2::V2_SERVER_VERSION;
|
||||
const CURRENT_DRIVER_VERSION: DriverVersion = versions::v2::V2_DRIVER_VERSION;
|
||||
}
|
||||
|
||||
pub struct SystemDatabaseV1;
|
||||
impl sdss::SimpleFileSpecV1 for SystemDatabaseV1 {
|
||||
type HeaderSpec = HeaderImplV2;
|
||||
const FILE_CLASS: FileClass = FileClass::EventLog;
|
||||
const FILE_SPECIFIER: FileSpecifier = FileSpecifier::GlobalNS;
|
||||
const FILE_SPECFIER_VERSION: FileSpecifierVersion = FileSpecifierVersion::__new(0);
|
||||
}
|
||||
|
||||
pub struct ModelDataBatchAofV1;
|
||||
impl sdss::SimpleFileSpecV1 for ModelDataBatchAofV1 {
|
||||
type HeaderSpec = HeaderImplV2;
|
||||
const FILE_CLASS: FileClass = FileClass::Batch;
|
||||
const FILE_SPECIFIER: FileSpecifier = FileSpecifier::ModelData;
|
||||
const FILE_SPECFIER_VERSION: FileSpecifierVersion = FileSpecifierVersion::__new(0);
|
||||
}
|
||||
Loading…
Reference in New Issue