From 843adbaf7e33f19babc96036fe6ad8d1fe155af9 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Fri, 19 Jan 2024 00:01:51 +0530 Subject: [PATCH] Refactor SDSSv1 primitives --- server/src/engine/core/index/key.rs | 2 +- server/src/engine/data/cell.rs | 2 +- server/src/engine/mem/mod.rs | 15 +----- server/src/engine/mem/unsafe_apis.rs | 53 +++++++++++++++++++ .../engine/storage/common/sdss/impls/mod.rs | 39 ++++++++++++++ .../common/sdss/{spec.rs => impls/v1/mod.rs} | 22 +++----- server/src/engine/storage/common/sdss/mod.rs | 4 +- server/src/engine/storage/v1/journal.rs | 14 +++-- server/src/engine/storage/v1/rw.rs | 6 +-- server/src/engine/storage/v1/spec.rs | 16 +++--- server/src/engine/storage/v1/sysdb.rs | 3 +- server/src/engine/storage/v2/spec.rs | 12 ++--- 12 files changed, 134 insertions(+), 54 deletions(-) create mode 100644 server/src/engine/mem/unsafe_apis.rs create mode 100644 server/src/engine/storage/common/sdss/impls/mod.rs rename server/src/engine/storage/common/sdss/{spec.rs => impls/v1/mod.rs} (96%) diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs index 801d5a88..351284f1 100644 --- a/server/src/engine/core/index/key.rs +++ b/server/src/engine/core/index/key.rs @@ -218,7 +218,7 @@ impl Drop for PrimaryIndexKey { unsafe { // UNSAFE(@ohsayan): Aliasing, sole owner and correct initialization let vdata = self.virtual_block_mut(); - mem::dealloc_array(vdata.as_mut_ptr(), vdata.len()); + mem::unsafe_apis::dealloc_array(vdata.as_mut_ptr(), vdata.len()); } } } diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs index 0c157a86..af67c50b 100644 --- a/server/src/engine/data/cell.rs +++ b/server/src/engine/data/cell.rs @@ -465,7 +465,7 @@ impl Drop for Datacell { TagClass::Str | TagClass::Bin => unsafe { // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class let (l, p) = self.load_word(); - engine::mem::dealloc_array::(p, l) + engine::mem::unsafe_apis::dealloc_array::(p, l) }, TagClass::List => unsafe { // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs index 4bb47e43..e06e13eb 100644 --- a/server/src/engine/mem/mod.rs +++ b/server/src/engine/mem/mod.rs @@ -31,6 +31,7 @@ mod rawslice; pub mod scanner; mod stackop; mod uarray; +pub mod unsafe_apis; mod vinline; mod word; // test @@ -47,20 +48,6 @@ pub use { vinline::VInline, word::{DwordNN, DwordQN, WordIO, ZERO_BLOCK}, }; -// imports -use std::alloc::{self, Layout}; - -pub unsafe fn dealloc_array(ptr: *mut T, l: usize) { - if l != 0 { - alloc::dealloc(ptr as *mut u8, Layout::array::(l).unwrap_unchecked()) - } -} - -pub unsafe fn memcpy(src: &[u8]) -> [u8; N] { - let mut dst = [0u8; N]; - src.as_ptr().copy_to_nonoverlapping(dst.as_mut_ptr(), N); - dst -} /// Native double pointer width (note, native != arch native, but host native) pub struct NativeDword([usize; 2]); diff --git a/server/src/engine/mem/unsafe_apis.rs b/server/src/engine/mem/unsafe_apis.rs new file mode 100644 index 00000000..2080137c --- /dev/null +++ b/server/src/engine/mem/unsafe_apis.rs @@ -0,0 +1,53 @@ +/* + * Created on Thu Jan 18 2024 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2024, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use std::alloc::{self, Layout}; + +pub unsafe fn alloc_layout(layout: Layout) -> *mut T { + let ptr = alloc::alloc(layout); + assert!(!ptr.is_null(), "malloc failed"); + ptr as _ +} + +pub unsafe fn alloc_array(l: usize) -> *mut T { + self::alloc_layout(Layout::array::(l).unwrap_unchecked()) +} + +pub unsafe fn dealloc_layout(ptr: *mut u8, layout: Layout) { + alloc::dealloc(ptr, layout) +} + +pub unsafe fn dealloc_array(ptr: *mut T, l: usize) { + if l != 0 { + self::dealloc_layout(ptr as *mut u8, Layout::array::(l).unwrap_unchecked()) + } +} + +pub unsafe fn memcpy(src: &[u8]) -> [u8; N] { + let mut dst = [0u8; N]; + src.as_ptr().copy_to_nonoverlapping(dst.as_mut_ptr(), N); + dst +} diff --git a/server/src/engine/storage/common/sdss/impls/mod.rs b/server/src/engine/storage/common/sdss/impls/mod.rs new file mode 100644 index 00000000..a7c59bdb --- /dev/null +++ b/server/src/engine/storage/common/sdss/impls/mod.rs @@ -0,0 +1,39 @@ +/* + * Created on Wed Jan 10 2024 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2024, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/*! + # SDSS spec + + This module provides traits and types to deal with the SDSS spec, especially headers. + + The static SDSS header block has a special segment that defines the header version which is static and will + never change across any versions. While the same isn't warranted for the rest of the header, it's exceedingly + unlikely that we'll ever change the static block. + + The only header that we currently use is [`v1::HeaderV1`]. +*/ + +pub mod v1; diff --git a/server/src/engine/storage/common/sdss/spec.rs b/server/src/engine/storage/common/sdss/impls/v1/mod.rs similarity index 96% rename from server/src/engine/storage/common/sdss/spec.rs rename to server/src/engine/storage/common/sdss/impls/v1/mod.rs index 6d65717a..7d9304f0 100644 --- a/server/src/engine/storage/common/sdss/spec.rs +++ b/server/src/engine/storage/common/sdss/impls/v1/mod.rs @@ -1,5 +1,5 @@ /* - * Created on Wed Jan 10 2024 + * Created on Thu Jan 18 2024 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -24,26 +24,20 @@ * */ -/*! - # SDSS spec - - This module provides traits and types to deal with the SDSS spec, especially headers. - - The static SDSS header block has a special segment that defines the header version which is static and will - never change across any versions. While the same isn't warranted for the rest of the header, it's exceedingly - unlikely that we'll ever change the static block. - - The only header that we currently use is [`HeaderV1`]. -*/ +//! # SDSS Spec v1 +//! +//! This is the first spec of SDSS. It is highly unlikely that this will ever change. Different types +//! and functions are defined here to deal with SDSSv1 files. +//! use { - super::super::{ + super::super::super::{ interface::fs_traits::{FileInterfaceRead, FileInterfaceWrite}, static_meta::{HostArch, HostEndian, HostOS, HostPointerWidth, SDSS_MAGIC_8B}, versions::{self, DriverVersion, FileSpecifierVersion, HeaderVersion, ServerVersion}, }, crate::{ - engine::{error::StorageError, mem::memcpy, RuntimeResult}, + engine::{error::StorageError, mem::unsafe_apis::memcpy, RuntimeResult}, util::os, }, std::{ diff --git a/server/src/engine/storage/common/sdss/mod.rs b/server/src/engine/storage/common/sdss/mod.rs index 56c9f396..25730e09 100644 --- a/server/src/engine/storage/common/sdss/mod.rs +++ b/server/src/engine/storage/common/sdss/mod.rs @@ -24,5 +24,5 @@ * */ -mod spec; -pub use spec::{FileSpecV1, HeaderV1, HeaderV1Enumeration, HeaderV1Spec, SimpleFileSpecV1}; +mod impls; +pub use impls::v1; diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs index 4b2045a4..e2518427 100644 --- a/server/src/engine/storage/v1/journal.rs +++ b/server/src/engine/storage/v1/journal.rs @@ -61,7 +61,7 @@ const CRC: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); pub fn open_or_create_journal< TA: JournalAdapter, Fs: FSInterface, - F: sdss::FileSpecV1, + F: sdss::v1::FileSpecV1, >( log_file_name: &str, gs: &TA::GlobalState, @@ -76,13 +76,21 @@ pub fn open_or_create_journal< )?)) } -pub fn create_journal>( +pub fn create_journal< + TA: JournalAdapter, + Fs: FSInterface, + F: sdss::v1::FileSpecV1, +>( log_file_name: &str, ) -> RuntimeResult> { JournalWriter::new(SDSSFileIO::create::(log_file_name)?, 0, true) } -pub fn load_journal>( +pub fn load_journal< + TA: JournalAdapter, + Fs: FSInterface, + F: sdss::v1::FileSpecV1, +>( log_file_name: &str, gs: &TA::GlobalState, ) -> RuntimeResult> { diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs index f2b21443..f0b75ba8 100644 --- a/server/src/engine/storage/v1/rw.rs +++ b/server/src/engine/storage/v1/rw.rs @@ -156,19 +156,19 @@ pub struct SDSSFileIO::File> { } impl SDSSFileIO { - pub fn open>( + pub fn open>( fpath: &str, ) -> RuntimeResult<(Self, F::Metadata)> { let mut f = Self::_new(Fs::fs_fopen_rw(fpath)?); let v = F::read_metadata(&mut f.f, ())?; Ok((f, v)) } - pub fn create>(fpath: &str) -> RuntimeResult { + pub fn create>(fpath: &str) -> RuntimeResult { let mut f = Self::_new(Fs::fs_fcreate_rw(fpath)?); F::write_metadata(&mut f.f, ())?; Ok(f) } - pub fn open_or_create_perm_rw>( + pub fn open_or_create_perm_rw>( fpath: &str, ) -> RuntimeResult> { match Fs::fs_fopen_or_create_rw(fpath)? { diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs index 07645d0d..042bc2b6 100644 --- a/server/src/engine/storage/v1/spec.rs +++ b/server/src/engine/storage/v1/spec.rs @@ -29,17 +29,17 @@ use crate::engine::storage::common::{ versions::{self, DriverVersion, FileSpecifierVersion, ServerVersion}, }; -pub(super) type Header = sdss::HeaderV1; +pub(super) type Header = sdss::v1::HeaderV1; #[derive(Debug)] pub(super) struct HeaderImplV1; -impl sdss::HeaderV1Spec for HeaderImplV1 { +impl sdss::v1::HeaderV1Spec for HeaderImplV1 { type FileClass = FileScope; type FileSpecifier = FileSpecifier; const CURRENT_SERVER_VERSION: ServerVersion = versions::v1::V1_SERVER_VERSION; const CURRENT_DRIVER_VERSION: DriverVersion = versions::v1::V1_DRIVER_VERSION; } -impl sdss::HeaderV1Enumeration for FileScope { +impl sdss::v1::HeaderV1Enumeration for FileScope { const MAX: u8 = FileScope::MAX; unsafe fn new(x: u8) -> Self { core::mem::transmute(x) @@ -48,7 +48,7 @@ impl sdss::HeaderV1Enumeration for FileScope { FileScope::value_u8(self) } } -impl sdss::HeaderV1Enumeration for FileSpecifier { +impl sdss::v1::HeaderV1Enumeration for FileSpecifier { const MAX: u8 = FileSpecifier::MAX; unsafe fn new(x: u8) -> Self { core::mem::transmute(x) @@ -84,7 +84,7 @@ pub enum FileSpecifier { #[cfg(test)] pub(super) struct TestFile; #[cfg(test)] -impl sdss::SimpleFileSpecV1 for TestFile { +impl sdss::v1::SimpleFileSpecV1 for TestFile { type HeaderSpec = HeaderImplV1; const FILE_CLASS: FileScope = FileScope::FlatmapData; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::TestTransactionLog; @@ -93,7 +93,7 @@ impl sdss::SimpleFileSpecV1 for TestFile { /// The file specification for the GNS transaction log (impl v1) pub(super) struct GNSTransactionLogV1; -impl sdss::SimpleFileSpecV1 for GNSTransactionLogV1 { +impl sdss::v1::SimpleFileSpecV1 for GNSTransactionLogV1 { type HeaderSpec = HeaderImplV1; const FILE_CLASS: FileScope = FileScope::Journal; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::GNSTxnLog; @@ -102,7 +102,7 @@ impl sdss::SimpleFileSpecV1 for GNSTransactionLogV1 { /// The file specification for a journal batch pub(super) struct DataBatchJournalV1; -impl sdss::SimpleFileSpecV1 for DataBatchJournalV1 { +impl sdss::v1::SimpleFileSpecV1 for DataBatchJournalV1 { type HeaderSpec = HeaderImplV1; const FILE_CLASS: FileScope = FileScope::DataBatch; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::TableDataBatch; @@ -111,7 +111,7 @@ impl sdss::SimpleFileSpecV1 for DataBatchJournalV1 { /// The file specification for the system db pub(super) struct SysDBV1; -impl sdss::SimpleFileSpecV1 for SysDBV1 { +impl sdss::v1::SimpleFileSpecV1 for SysDBV1 { type HeaderSpec = HeaderImplV1; const FILE_CLASS: FileScope = FileScope::FlatmapData; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::SysDB; diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs index 3b6d49d9..9a578f44 100644 --- a/server/src/engine/storage/v1/sysdb.rs +++ b/server/src/engine/storage/v1/sysdb.rs @@ -181,8 +181,7 @@ impl SystemStore { Ok((slf, state)) } fn _restore(mut f: SDSSFileIO, run_mode: ConfigMode) -> RuntimeResult { - let mut sysdb_data = - inf::dec::dec_dict_full::(&f.read_full()?)?; + let mut sysdb_data = inf::dec::dec_dict_full::(&f.read_full()?)?; // get our auth and sys stores let mut auth_store = rkey( &mut sysdb_data, diff --git a/server/src/engine/storage/v2/spec.rs b/server/src/engine/storage/v2/spec.rs index 8e7ff45a..d6f561cb 100644 --- a/server/src/engine/storage/v2/spec.rs +++ b/server/src/engine/storage/v2/spec.rs @@ -26,7 +26,7 @@ use { crate::engine::storage::common::{ - sdss::{self, HeaderV1Spec}, + sdss, versions::{self, DriverVersion, FileSpecifierVersion, ServerVersion}, }, std::mem::transmute, @@ -47,7 +47,7 @@ pub enum FileSpecifier { ModelData = 1, } -impl sdss::HeaderV1Enumeration for FileClass { +impl sdss::v1::HeaderV1Enumeration for FileClass { const MAX: u8 = FileClass::MAX; unsafe fn new(x: u8) -> Self { transmute(x) @@ -57,7 +57,7 @@ impl sdss::HeaderV1Enumeration for FileClass { } } -impl sdss::HeaderV1Enumeration for FileSpecifier { +impl sdss::v1::HeaderV1Enumeration for FileSpecifier { const MAX: u8 = FileSpecifier::MAX; unsafe fn new(x: u8) -> Self { transmute(x) @@ -68,7 +68,7 @@ impl sdss::HeaderV1Enumeration for FileSpecifier { } pub struct HeaderImplV2; -impl HeaderV1Spec for HeaderImplV2 { +impl sdss::v1::HeaderV1Spec for HeaderImplV2 { type FileClass = FileClass; type FileSpecifier = FileSpecifier; const CURRENT_SERVER_VERSION: ServerVersion = versions::v2::V2_SERVER_VERSION; @@ -76,7 +76,7 @@ impl HeaderV1Spec for HeaderImplV2 { } pub struct SystemDatabaseV1; -impl sdss::SimpleFileSpecV1 for SystemDatabaseV1 { +impl sdss::v1::SimpleFileSpecV1 for SystemDatabaseV1 { type HeaderSpec = HeaderImplV2; const FILE_CLASS: FileClass = FileClass::EventLog; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::GlobalNS; @@ -84,7 +84,7 @@ impl sdss::SimpleFileSpecV1 for SystemDatabaseV1 { } pub struct ModelDataBatchAofV1; -impl sdss::SimpleFileSpecV1 for ModelDataBatchAofV1 { +impl sdss::v1::SimpleFileSpecV1 for ModelDataBatchAofV1 { type HeaderSpec = HeaderImplV2; const FILE_CLASS: FileClass = FileClass::Batch; const FILE_SPECIFIER: FileSpecifier = FileSpecifier::ModelData;