Add new fs abstraction
Sayan Nandan
7 months ago
parent
1cb35b6b36
commit
0573e80992
@ -0,0 +1,568 @@
|
|||||||
|
/*
|
||||||
|
* Created on Thu Feb 29 2024
|
||||||
|
*
|
||||||
|
* This file is a part of Skytable
|
||||||
|
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
||||||
|
* NoSQL database written by Sayan Nandan ("the Author") with the
|
||||||
|
* vision to provide flexibility in data modelling without compromising
|
||||||
|
* on performance, queryability or scalability.
|
||||||
|
*
|
||||||
|
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
|
/*
|
||||||
|
file system
|
||||||
|
*/
|
||||||
|
|
||||||
|
use std::io::BufWriter;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::vfs::{VFileDescriptor, VirtualFS};
|
||||||
|
use {
|
||||||
|
crate::IoResult,
|
||||||
|
std::{
|
||||||
|
fs as std_fs,
|
||||||
|
io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
pub struct FileSystem {}
|
||||||
|
|
||||||
|
impl Default for FileSystem {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::instance()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, PartialEq, Clone, Copy)]
|
||||||
|
pub enum FSContext {
|
||||||
|
Local,
|
||||||
|
Virtual,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileSystem {
|
||||||
|
pub fn instance() -> Self {
|
||||||
|
Self {}
|
||||||
|
}
|
||||||
|
fn context() -> FSContext {
|
||||||
|
local! { static CTX: FSContext = FSContext::Virtual; }
|
||||||
|
local_ref!(CTX, |ctx| *ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileSystem {
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn read(path: &str) -> IoResult<Vec<u8>> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().read().get_data(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::read(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn create_dir(path: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_create_dir(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::create_dir(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn create_dir_all(path: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_create_dir_all(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::create_dir_all(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn remove_dir(path: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_delete_dir(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::remove_dir(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn remove_dir_all(path: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_delete_dir_all(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::remove_dir_all(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn remove_file(path: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_remove_file(path),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::remove_file(path)
|
||||||
|
}
|
||||||
|
#[inline(always)]
|
||||||
|
pub fn rename(from: &str, to: &str) -> IoResult<()> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match Self::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => return VirtualFS::instance().write().fs_rename(from, to),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std_fs::rename(from, to)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
file traits
|
||||||
|
*/
|
||||||
|
|
||||||
|
pub trait FileRead {
|
||||||
|
fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()>;
|
||||||
|
fn fread_exact_block<const N: usize>(&mut self) -> IoResult<[u8; N]> {
|
||||||
|
let mut blk = [0; N];
|
||||||
|
self.fread_exact(&mut blk).map(|_| blk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait FileWrite {
|
||||||
|
fn fwrite(&mut self, buf: &[u8]) -> IoResult<u64>;
|
||||||
|
fn fwrite_all(&mut self, buf: &[u8]) -> IoResult<()> {
|
||||||
|
self.fwrite_all_count(buf).1
|
||||||
|
}
|
||||||
|
fn fwrite_all_count(&mut self, buf: &[u8]) -> (u64, IoResult<()>) {
|
||||||
|
let len = buf.len() as u64;
|
||||||
|
let mut written = 0;
|
||||||
|
while written != len {
|
||||||
|
match self.fwrite(buf) {
|
||||||
|
Ok(0) => {
|
||||||
|
return (
|
||||||
|
written,
|
||||||
|
Err(Error::new(
|
||||||
|
ErrorKind::WriteZero,
|
||||||
|
format!("could only write {} of {} bytes", written, buf.len()),
|
||||||
|
)
|
||||||
|
.into()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Ok(n) => written += n,
|
||||||
|
Err(e) => return (written, Err(e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(written, Ok(()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait FileWriteExt {
|
||||||
|
fn fsync_all(&mut self) -> IoResult<()>;
|
||||||
|
fn fsync_data(&mut self) -> IoResult<()>;
|
||||||
|
fn f_truncate(&mut self, new_size: u64) -> IoResult<()>;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait FileExt {
|
||||||
|
fn f_len(&self) -> IoResult<u64>;
|
||||||
|
fn f_cursor(&mut self) -> IoResult<u64>;
|
||||||
|
fn f_seek_start(&mut self, offset: u64) -> IoResult<()>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
file impls
|
||||||
|
*/
|
||||||
|
|
||||||
|
impl FileWrite for File {
|
||||||
|
fn fwrite(&mut self, buf: &[u8]) -> IoResult<u64> {
|
||||||
|
self.f.fwrite(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileRead for File {
|
||||||
|
fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> {
|
||||||
|
self.f.fread_exact(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileWriteExt for File {
|
||||||
|
fn fsync_all(&mut self) -> IoResult<()> {
|
||||||
|
self.f.fsync_all()
|
||||||
|
}
|
||||||
|
fn fsync_data(&mut self) -> IoResult<()> {
|
||||||
|
self.f.fsync_data()
|
||||||
|
}
|
||||||
|
fn f_truncate(&mut self, new_size: u64) -> IoResult<()> {
|
||||||
|
self.f.f_truncate(new_size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileExt for File {
|
||||||
|
fn f_len(&self) -> IoResult<u64> {
|
||||||
|
self.f.f_len()
|
||||||
|
}
|
||||||
|
fn f_cursor(&mut self) -> IoResult<u64> {
|
||||||
|
self.f.f_cursor()
|
||||||
|
}
|
||||||
|
fn f_seek_start(&mut self, offset: u64) -> IoResult<()> {
|
||||||
|
self.f.f_seek_start(offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
impls for local file
|
||||||
|
*/
|
||||||
|
|
||||||
|
trait LocalFile {
|
||||||
|
fn _mut(&mut self) -> &mut std_fs::File;
|
||||||
|
fn _ref(&self) -> &std_fs::File;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalFile for BufReader<std_fs::File> {
|
||||||
|
fn _mut(&mut self) -> &mut std_fs::File {
|
||||||
|
self.get_mut()
|
||||||
|
}
|
||||||
|
fn _ref(&self) -> &std_fs::File {
|
||||||
|
self.get_ref()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalFile for std_fs::File {
|
||||||
|
fn _mut(&mut self) -> &mut std_fs::File {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
fn _ref(&self) -> &std_fs::File {
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<W: Write> FileWrite for W {
|
||||||
|
fn fwrite(&mut self, buf: &[u8]) -> IoResult<u64> {
|
||||||
|
self.write(buf).map(|x| x as u64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<R: Read> FileRead for R {
|
||||||
|
fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> {
|
||||||
|
self.read_exact(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Lf: LocalFile> FileWriteExt for Lf {
|
||||||
|
fn fsync_all(&mut self) -> IoResult<()> {
|
||||||
|
self._mut().sync_all()
|
||||||
|
}
|
||||||
|
fn fsync_data(&mut self) -> IoResult<()> {
|
||||||
|
self._mut().sync_data()
|
||||||
|
}
|
||||||
|
fn f_truncate(&mut self, new_size: u64) -> IoResult<()> {
|
||||||
|
self._mut().set_len(new_size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<Lf: LocalFile> FileExt for Lf {
|
||||||
|
fn f_len(&self) -> IoResult<u64> {
|
||||||
|
self._ref().metadata().map(|md| md.len())
|
||||||
|
}
|
||||||
|
fn f_cursor(&mut self) -> IoResult<u64> {
|
||||||
|
self._mut().stream_position()
|
||||||
|
}
|
||||||
|
fn f_seek_start(&mut self, offset: u64) -> IoResult<()> {
|
||||||
|
self._mut().seek(SeekFrom::Start(offset)).map(|_| ())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
impls for vfile
|
||||||
|
*/
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<Lf: FileWrite> FileWrite for AnyFile<Lf> {
|
||||||
|
fn fwrite(&mut self, buf: &[u8]) -> IoResult<u64> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.fwrite(buf),
|
||||||
|
Self::Virtual(vf) => vf.get_mut(&mut VirtualFS::instance().write()).fwrite(buf),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<Lf: FileRead> FileRead for AnyFile<Lf> {
|
||||||
|
fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.fread_exact(buf),
|
||||||
|
Self::Virtual(vf) => vf
|
||||||
|
.get_mut(&mut VirtualFS::instance().write())
|
||||||
|
.fread_exact(buf),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<Lf: FileWriteExt> FileWriteExt for AnyFile<Lf> {
|
||||||
|
fn fsync_all(&mut self) -> IoResult<()> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.fsync_all(),
|
||||||
|
Self::Virtual(_) => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn fsync_data(&mut self) -> IoResult<()> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.fsync_data(),
|
||||||
|
Self::Virtual(_) => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn f_truncate(&mut self, new_size: u64) -> IoResult<()> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.f_truncate(new_size),
|
||||||
|
Self::Virtual(vf) => vf
|
||||||
|
.get_mut(&mut VirtualFS::instance().write())
|
||||||
|
.truncate(new_size),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<Lf: FileExt> FileExt for AnyFile<Lf> {
|
||||||
|
fn f_len(&self) -> IoResult<u64> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.f_len(),
|
||||||
|
Self::Virtual(vf) => vf.get_ref(&VirtualFS::instance().read()).length(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn f_cursor(&mut self) -> IoResult<u64> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.f_cursor(),
|
||||||
|
Self::Virtual(vf) => vf.get_ref(&VirtualFS::instance().read()).cursor(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn f_seek_start(&mut self, offset: u64) -> IoResult<()> {
|
||||||
|
match self {
|
||||||
|
Self::Local(lf) => lf.f_seek_start(offset),
|
||||||
|
Self::Virtual(vf) => vf
|
||||||
|
.get_mut(&mut VirtualFS::instance().write())
|
||||||
|
.seek_from_start(offset),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
file abstraction
|
||||||
|
*/
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
enum AnyFile<Lf = std_fs::File> {
|
||||||
|
Local(Lf),
|
||||||
|
Virtual(VFileDescriptor),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct File {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile,
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: std_fs::File,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl File {
|
||||||
|
pub fn open(path: &str) -> IoResult<Self> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match FileSystem::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => {
|
||||||
|
return VirtualFS::instance()
|
||||||
|
.write()
|
||||||
|
.fs_fopen_rw(path)
|
||||||
|
.map(|f| Self {
|
||||||
|
f: AnyFile::Virtual(f),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let file = std_fs::File::options().read(true).write(true).open(path)?;
|
||||||
|
Ok(Self {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile::Local(file),
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: file,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
pub fn create(path: &str) -> IoResult<Self> {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match FileSystem::context() {
|
||||||
|
FSContext::Local => {}
|
||||||
|
FSContext::Virtual => {
|
||||||
|
return VirtualFS::instance()
|
||||||
|
.write()
|
||||||
|
.fs_fcreate_rw(path)
|
||||||
|
.map(|f| Self {
|
||||||
|
f: AnyFile::Virtual(f),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let file = std_fs::File::options()
|
||||||
|
.create_new(true)
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.open(path)?;
|
||||||
|
Ok(Self {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile::Local(file),
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: file,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
pub fn into_buffered_reader(self) -> BufferedReader {
|
||||||
|
BufferedReader::new(self.f)
|
||||||
|
}
|
||||||
|
pub fn into_buffered_writer(self) -> BufferedWriter {
|
||||||
|
BufferedWriter::new(self.f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
buffered readers and writers
|
||||||
|
*/
|
||||||
|
|
||||||
|
pub struct BufferedReader {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile<BufReader<std_fs::File>>,
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: BufReader<std_fs::File>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BufferedReader {
|
||||||
|
fn new(#[cfg(test)] f: AnyFile<std_fs::File>, #[cfg(not(test))] f: std_fs::File) -> Self {
|
||||||
|
Self {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: match f {
|
||||||
|
AnyFile::Local(lf) => AnyFile::Local(BufReader::new(lf)),
|
||||||
|
AnyFile::Virtual(vf) => AnyFile::Virtual(vf),
|
||||||
|
},
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: BufReader::new(f),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn into_inner(self) -> File {
|
||||||
|
File {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: match self.f {
|
||||||
|
AnyFile::Local(lf) => AnyFile::Local(lf.into_inner()),
|
||||||
|
AnyFile::Virtual(vf) => AnyFile::Virtual(vf),
|
||||||
|
},
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: self.f.into_inner(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileRead for BufferedReader {
|
||||||
|
fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> {
|
||||||
|
self.f.fread_exact(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileExt for BufferedReader {
|
||||||
|
fn f_len(&self) -> IoResult<u64> {
|
||||||
|
self.f.f_len()
|
||||||
|
}
|
||||||
|
fn f_cursor(&mut self) -> IoResult<u64> {
|
||||||
|
self.f.f_cursor()
|
||||||
|
}
|
||||||
|
fn f_seek_start(&mut self, offset: u64) -> IoResult<()> {
|
||||||
|
self.f.f_seek_start(offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct BufferedWriter {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile<BufWriter<std_fs::File>>,
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: BufWriter<std_fs::File>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BufferedWriter {
|
||||||
|
pub fn into_inner(self) -> IoResult<File> {
|
||||||
|
let mut local;
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match self.f {
|
||||||
|
AnyFile::Local(lf) => local = lf,
|
||||||
|
AnyFile::Virtual(vf) => {
|
||||||
|
return Ok(File {
|
||||||
|
f: AnyFile::Virtual(vf),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(not(test))]
|
||||||
|
{
|
||||||
|
local = self.f;
|
||||||
|
}
|
||||||
|
local.flush()?;
|
||||||
|
let local = local.into_inner().unwrap();
|
||||||
|
Ok(File {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: AnyFile::Local(local),
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: local,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
pub fn flush(&mut self) -> IoResult<()> {
|
||||||
|
let local;
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
match self.f {
|
||||||
|
AnyFile::Local(ref mut l) => local = l,
|
||||||
|
AnyFile::Virtual(_) => return Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(not(test))]
|
||||||
|
{
|
||||||
|
local = &mut self.f;
|
||||||
|
}
|
||||||
|
local.flush()
|
||||||
|
}
|
||||||
|
fn new(#[cfg(test)] f: AnyFile<std_fs::File>, #[cfg(not(test))] f: std_fs::File) -> Self {
|
||||||
|
Self {
|
||||||
|
#[cfg(test)]
|
||||||
|
f: match f {
|
||||||
|
AnyFile::Local(lf) => AnyFile::Local(BufWriter::new(lf)),
|
||||||
|
AnyFile::Virtual(vf) => AnyFile::Virtual(vf),
|
||||||
|
},
|
||||||
|
#[cfg(not(test))]
|
||||||
|
f: BufWriter::new(f),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,205 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Sun Jan 07 2024
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
super::fs_traits::{
|
|
||||||
FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, FileInterfaceRead,
|
|
||||||
FileInterfaceWrite, FileInterfaceWriteExt, FileOpen,
|
|
||||||
},
|
|
||||||
crate::engine::RuntimeResult,
|
|
||||||
std::{
|
|
||||||
fs::{self, File},
|
|
||||||
io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/*
|
|
||||||
local fs impls
|
|
||||||
*/
|
|
||||||
|
|
||||||
/// A type representing the host's local filesystem (or atleast where our data directory is)
|
|
||||||
pub struct LocalFS;
|
|
||||||
|
|
||||||
fn cvt<T, E1, E2: From<E1>>(r: Result<T, E1>) -> Result<T, E2> {
|
|
||||||
r.map_err(Into::into)
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FSInterface for LocalFS {
|
|
||||||
type File = File;
|
|
||||||
fn fs_remove_file(fpath: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::remove_file(fpath))
|
|
||||||
}
|
|
||||||
fn fs_rename(from: &str, to: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::rename(from, to))
|
|
||||||
}
|
|
||||||
fn fs_create_dir(fpath: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::create_dir(fpath))
|
|
||||||
}
|
|
||||||
fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::create_dir_all(fpath))
|
|
||||||
}
|
|
||||||
fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::remove_dir(fpath))
|
|
||||||
}
|
|
||||||
fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> {
|
|
||||||
cvt(fs::remove_dir_all(fpath))
|
|
||||||
}
|
|
||||||
fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<super::fs_traits::FileOpen<Self::File>> {
|
|
||||||
let r = || -> Result<_, std::io::Error> {
|
|
||||||
let f = File::options()
|
|
||||||
.create(true)
|
|
||||||
.read(true)
|
|
||||||
.write(true)
|
|
||||||
.open(fpath)?;
|
|
||||||
let md = f.metadata()?;
|
|
||||||
if md.len() == 0 {
|
|
||||||
Ok(FileOpen::Created(f))
|
|
||||||
} else {
|
|
||||||
Ok(FileOpen::Existing(f))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
cvt(r())
|
|
||||||
}
|
|
||||||
fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
|
||||||
let f = File::options().read(true).write(true).open(fpath)?;
|
|
||||||
Ok(f)
|
|
||||||
}
|
|
||||||
fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File> {
|
|
||||||
let f = File::options()
|
|
||||||
.create_new(true)
|
|
||||||
.read(true)
|
|
||||||
.write(true)
|
|
||||||
.open(fpath)?;
|
|
||||||
Ok(f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
common impls for files
|
|
||||||
*/
|
|
||||||
|
|
||||||
impl<R: Read> FileInterfaceRead for R {
|
|
||||||
fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
|
|
||||||
cvt(self.read_exact(buf))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<W: Write> FileInterfaceWrite for W {
|
|
||||||
fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult<u64> {
|
|
||||||
cvt(self.write(buf).map(|v| v as _))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
local file impls
|
|
||||||
*/
|
|
||||||
|
|
||||||
impl FileInterface for File {
|
|
||||||
type BufReader = BufReader<Self>;
|
|
||||||
type BufWriter = BufWriter<Self>;
|
|
||||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
|
|
||||||
Ok(BufReader::new(self))
|
|
||||||
}
|
|
||||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
|
|
||||||
Ok(BufWriter::new(self))
|
|
||||||
}
|
|
||||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
|
|
||||||
Ok(r.into_inner())
|
|
||||||
}
|
|
||||||
fn downgrade_writer(mut r: Self::BufWriter) -> RuntimeResult<Self> {
|
|
||||||
// TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this?
|
|
||||||
r.flush()?;
|
|
||||||
let (me, err) = r.into_parts();
|
|
||||||
match err {
|
|
||||||
Ok(x) if x.is_empty() => Ok(me),
|
|
||||||
Ok(_) | Err(_) => {
|
|
||||||
return Err(std::io::Error::new(
|
|
||||||
std::io::ErrorKind::Other,
|
|
||||||
"failed to flush data from buffer into sink",
|
|
||||||
)
|
|
||||||
.into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A trait for handling wrappers of [`std::fs::File`]
|
|
||||||
trait AsLocalFile {
|
|
||||||
fn file(&self) -> &File;
|
|
||||||
fn file_mut(&mut self) -> &mut File;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AsLocalFile for File {
|
|
||||||
fn file(&self) -> &File {
|
|
||||||
self
|
|
||||||
}
|
|
||||||
fn file_mut(&mut self) -> &mut File {
|
|
||||||
self
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AsLocalFile for BufReader<File> {
|
|
||||||
fn file(&self) -> &File {
|
|
||||||
self.get_ref()
|
|
||||||
}
|
|
||||||
fn file_mut(&mut self) -> &mut File {
|
|
||||||
self.get_mut()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl AsLocalFile for BufWriter<File> {
|
|
||||||
fn file(&self) -> &File {
|
|
||||||
self.get_ref()
|
|
||||||
}
|
|
||||||
fn file_mut(&mut self) -> &mut File {
|
|
||||||
self.get_mut()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FileInterfaceBufWrite for BufWriter<File> {
|
|
||||||
fn sync_write_cache(&mut self) -> RuntimeResult<()> {
|
|
||||||
// TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this?
|
|
||||||
cvt(self.flush())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<F: AsLocalFile> FileInterfaceExt for F {
|
|
||||||
fn fext_length(&self) -> RuntimeResult<u64> {
|
|
||||||
Ok(self.file().metadata()?.len())
|
|
||||||
}
|
|
||||||
fn fext_cursor(&mut self) -> RuntimeResult<u64> {
|
|
||||||
cvt(self.file_mut().stream_position())
|
|
||||||
}
|
|
||||||
fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> {
|
|
||||||
cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FileInterfaceWriteExt for File {
|
|
||||||
fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
|
|
||||||
cvt(self.set_len(to))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,176 +0,0 @@
|
|||||||
/*
|
|
||||||
* Created on Sun Jan 07 2024
|
|
||||||
*
|
|
||||||
* This file is a part of Skytable
|
|
||||||
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
|
|
||||||
* NoSQL database written by Sayan Nandan ("the Author") with the
|
|
||||||
* vision to provide flexibility in data modelling without compromising
|
|
||||||
* on performance, queryability or scalability.
|
|
||||||
*
|
|
||||||
* Copyright (c) 2024, Sayan Nandan <nandansayan@outlook.com>
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
use {
|
|
||||||
crate::engine::RuntimeResult,
|
|
||||||
std::io::{Error as IoError, ErrorKind as IoErrorKind},
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq)]
|
|
||||||
/// Result of opening a file
|
|
||||||
/// - Created: newly created file
|
|
||||||
/// - Existing: existing file that was reopened
|
|
||||||
pub enum FileOpen<CF, EF = CF> {
|
|
||||||
/// new file
|
|
||||||
Created(CF),
|
|
||||||
/// existing file
|
|
||||||
Existing(EF),
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait FSInterface {
|
|
||||||
// settings
|
|
||||||
/// set to false if the file system is a special device like `/dev/null`
|
|
||||||
const NOT_NULL: bool = true;
|
|
||||||
// types
|
|
||||||
/// the file type that is returned by this file system
|
|
||||||
type File: FileInterface;
|
|
||||||
// functions
|
|
||||||
/// Remove a file
|
|
||||||
fn fs_remove_file(fpath: &str) -> RuntimeResult<()>;
|
|
||||||
/// Rename a file
|
|
||||||
fn fs_rename(from: &str, to: &str) -> RuntimeResult<()>;
|
|
||||||
/// Create a directory
|
|
||||||
fn fs_create_dir(fpath: &str) -> RuntimeResult<()>;
|
|
||||||
/// Create a directory and all corresponding path components
|
|
||||||
fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()>;
|
|
||||||
/// Delete a directory
|
|
||||||
fn fs_delete_dir(fpath: &str) -> RuntimeResult<()>;
|
|
||||||
/// Delete a directory and recursively remove all (if any) children
|
|
||||||
fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()>;
|
|
||||||
/// Open or create a file in R/W mode
|
|
||||||
///
|
|
||||||
/// This will:
|
|
||||||
/// - Create a file if it doesn't exist
|
|
||||||
/// - Open a file it it does exist
|
|
||||||
fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<FileOpen<Self::File>>;
|
|
||||||
/// Open an existing file
|
|
||||||
fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File>;
|
|
||||||
/// Create a new file
|
|
||||||
fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// File interface definition
|
|
||||||
pub trait FileInterface:
|
|
||||||
FileInterfaceRead + FileInterfaceWrite + FileInterfaceWriteExt + FileInterfaceExt + Sized
|
|
||||||
{
|
|
||||||
/// A buffered reader implementation
|
|
||||||
type BufReader: FileInterfaceRead + FileInterfaceExt;
|
|
||||||
/// A buffered writer implementation
|
|
||||||
type BufWriter: FileInterfaceBufWrite;
|
|
||||||
/// Get a buffered reader for this file
|
|
||||||
fn upgrade_to_buffered_reader(self) -> RuntimeResult<Self::BufReader>;
|
|
||||||
/// Get a buffered writer for this file
|
|
||||||
fn upgrade_to_buffered_writer(self) -> RuntimeResult<Self::BufWriter>;
|
|
||||||
/// Get the file back from the buffered reader
|
|
||||||
fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self>;
|
|
||||||
/// Get the file back from the buffered writer
|
|
||||||
fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult<Self>;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait FileInterfaceBufWrite: FileInterfaceWrite + FileInterfaceExt {
|
|
||||||
fn sync_write_cache(&mut self) -> RuntimeResult<()>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Readable object
|
|
||||||
pub trait FileInterfaceRead {
|
|
||||||
/// Read in a block of the exact given length
|
|
||||||
fn fread_exact_block<const N: usize>(&mut self) -> RuntimeResult<[u8; N]> {
|
|
||||||
let mut ret = [0u8; N];
|
|
||||||
self.fread_exact(&mut ret)?;
|
|
||||||
Ok(ret)
|
|
||||||
}
|
|
||||||
/// Read in `n` bytes to fill the given buffer
|
|
||||||
fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Writable object
|
|
||||||
pub trait FileInterfaceWrite {
|
|
||||||
/// Attempt to write the buffer into this object, returning the number of bytes that were
|
|
||||||
/// written. It is **NOT GUARANTEED THAT ALL DATA WILL BE WRITTEN**
|
|
||||||
fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult<u64>;
|
|
||||||
/// Attempt to write the entire buffer into this object, returning the number of bytes written
|
|
||||||
///
|
|
||||||
/// It is guaranteed that if the [`Result`] returned is [`Ok(())`], then the entire buffer was
|
|
||||||
/// written to disk.
|
|
||||||
fn fwrite_all_count(&mut self, buf: &[u8]) -> (u64, RuntimeResult<()>) {
|
|
||||||
let len = buf.len() as u64;
|
|
||||||
let mut written = 0;
|
|
||||||
while written != len {
|
|
||||||
match self.fwrite(buf) {
|
|
||||||
Ok(0) => {
|
|
||||||
return (
|
|
||||||
written,
|
|
||||||
Err(IoError::new(
|
|
||||||
IoErrorKind::WriteZero,
|
|
||||||
format!("could only write {} of {} bytes", written, buf.len()),
|
|
||||||
)
|
|
||||||
.into()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
Ok(n) => written += n,
|
|
||||||
Err(e) => return (written, Err(e)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
(written, Ok(()))
|
|
||||||
}
|
|
||||||
/// Attempt to write the entire buffer into this object
|
|
||||||
///
|
|
||||||
/// If this return [`Ok(())`] then it is guaranteed that all bytes have been written
|
|
||||||
fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()> {
|
|
||||||
self.fwrite_all_count(buf).1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Advanced write traits
|
|
||||||
pub trait FileInterfaceWriteExt {
|
|
||||||
/// Sync data and metadata for this file
|
|
||||||
fn fwext_sync_all(&mut self) -> RuntimeResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
/// Sync data for this file
|
|
||||||
fn fwext_sync_data(&mut self) -> RuntimeResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
/// Sync meta for this file
|
|
||||||
fn fwext_sync_meta(&mut self) -> RuntimeResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
/// Truncate the size of the file to the given size
|
|
||||||
///
|
|
||||||
/// - If `to` > actual file length: the file is zero padded to fill `to - len`
|
|
||||||
/// - If `to` < actual file length: the file is trimmed to the size `to`
|
|
||||||
fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Advanced file access
|
|
||||||
pub trait FileInterfaceExt {
|
|
||||||
/// Returns the length of the file
|
|
||||||
fn fext_length(&self) -> RuntimeResult<u64>;
|
|
||||||
/// Returns the current cursor position of the file
|
|
||||||
fn fext_cursor(&mut self) -> RuntimeResult<u64>;
|
|
||||||
/// Seek by `from` bytes from the start of the file
|
|
||||||
fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()>;
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue