From e126c826f947677b656bf0fc9e250e1a67be92c2 Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Thu, 25 May 2023 18:26:46 +0000 Subject: [PATCH] Add new dynamic record definition and remove rw --- .../src/engine/storage/v1/header_impl/dr.rs | 373 ++++++++++++++++++ .../v1/{header_impl.rs => header_impl/gr.rs} | 276 +------------ .../src/engine/storage/v1/header_impl/mod.rs | 266 +++++++++++++ .../src/engine/storage/v1/header_impl/sr.rs | 68 ++++ server/src/engine/storage/v1/rw.rs | 188 --------- 5 files changed, 727 insertions(+), 444 deletions(-) create mode 100644 server/src/engine/storage/v1/header_impl/dr.rs rename server/src/engine/storage/v1/{header_impl.rs => header_impl/gr.rs} (64%) create mode 100644 server/src/engine/storage/v1/header_impl/mod.rs create mode 100644 server/src/engine/storage/v1/header_impl/sr.rs diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs new file mode 100644 index 00000000..1f530ec9 --- /dev/null +++ b/server/src/engine/storage/v1/header_impl/dr.rs @@ -0,0 +1,373 @@ +/* + * Created on Thu May 25 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::{ + mem::ByteStack, + storage::{ + header::{HostArch, HostEndian, HostOS, HostPointerWidth}, + v1::header_impl::FileSpecifierVersion, + versions::{DriverVersion, ServerVersion}, + }, +}; + +/* + Dynamic record (1/2): Host signature + --- + - 8B: Server version + - 8B: Driver version + - 4B: File specifier ID + - 1B: Endian + - 1B: Pointer width + - 1B: Arch + - 1B: OS +*/ + +#[derive(Debug, PartialEq)] +pub struct DRHostSignature { + server_version: ServerVersion, + driver_version: DriverVersion, + file_specifier_version: FileSpecifierVersion, + endian: HostEndian, + ptr_width: HostPointerWidth, + arch: HostArch, + os: HostOS, +} + +impl DRHostSignature { + pub const fn new( + server_version: ServerVersion, + driver_version: DriverVersion, + file_specifier_version: FileSpecifierVersion, + endian: HostEndian, + ptr_width: HostPointerWidth, + arch: HostArch, + os: HostOS, + ) -> Self { + Self { + server_version, + driver_version, + file_specifier_version, + endian, + ptr_width, + arch, + os, + } + } + pub const fn server_version(&self) -> ServerVersion { + self.server_version + } + pub const fn driver_version(&self) -> DriverVersion { + self.driver_version + } + pub const fn file_specifier_version(&self) -> FileSpecifierVersion { + self.file_specifier_version + } + pub const fn endian(&self) -> HostEndian { + self.endian + } + pub const fn ptr_width(&self) -> HostPointerWidth { + self.ptr_width + } + pub const fn arch(&self) -> HostArch { + self.arch + } + pub const fn os(&self) -> HostOS { + self.os + } + pub const fn encoded(&self) -> DRHostSignatureRaw { + DRHostSignatureRaw::new_full( + self.server_version(), + self.driver_version(), + self.file_specifier_version(), + self.endian(), + self.ptr_width(), + self.arch(), + self.os(), + ) + } +} + +#[derive(Debug, PartialEq)] +pub struct DRHostSignatureRaw { + data: ByteStack<24>, +} + +impl DRHostSignatureRaw { + const DRHS_OFFSET_P0: usize = 0; + const DRHS_OFFSET_P1: usize = sizeof!(u64); + const DRHS_OFFSET_P2: usize = Self::DRHS_OFFSET_P1 + sizeof!(u64); + const DRHS_OFFSET_P3: usize = Self::DRHS_OFFSET_P2 + sizeof!(u32); + const DRHS_OFFSET_P4: usize = Self::DRHS_OFFSET_P3 + 1; + const DRHS_OFFSET_P5: usize = Self::DRHS_OFFSET_P4 + 1; + const DRHS_OFFSET_P6: usize = Self::DRHS_OFFSET_P5 + 1; + const _ENSURE: () = assert!(Self::DRHS_OFFSET_P6 == sizeof!(Self) - 1); + pub const fn new( + server_version: ServerVersion, + driver_version: DriverVersion, + file_specifier_id: FileSpecifierVersion, + ) -> Self { + Self::new_full( + server_version, + driver_version, + file_specifier_id, + HostEndian::new(), + HostPointerWidth::new(), + HostArch::new(), + HostOS::new(), + ) + } + pub const fn new_full( + server_version: ServerVersion, + driver_version: DriverVersion, + file_specifier_id: FileSpecifierVersion, + endian: HostEndian, + ptr_width: HostPointerWidth, + arch: HostArch, + os: HostOS, + ) -> Self { + let _ = Self::_ENSURE; + let bytes: [u8; 24] = unsafe { + let [qw_a, qw_b]: [u64; 2] = core::mem::transmute([ + server_version.little_endian(), + driver_version.little_endian(), + ]); + let dw: u32 = core::mem::transmute([ + endian.value_u8(), + ptr_width.value_u8(), + arch.value_u8(), + os.value_u8(), + ]); + let qw_c: u64 = core::mem::transmute([(file_specifier_id.0.to_le(), dw.to_le())]); + core::mem::transmute([qw_a, qw_b, qw_c]) + }; + Self { + data: ByteStack::new(bytes), + } + } +} + +impl DRHostSignatureRaw { + pub const fn read_p0_server_version(&self) -> ServerVersion { + ServerVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P0)) + } + pub const fn read_p1_driver_version(&self) -> DriverVersion { + DriverVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P1)) + } + pub const fn read_p2_file_specifier_id(&self) -> FileSpecifierVersion { + FileSpecifierVersion::__new(self.data.read_dword(Self::DRHS_OFFSET_P2)) + } + pub const fn read_p3_endian(&self) -> HostEndian { + HostEndian::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P3)) + } + pub const fn read_p4_pointer_width(&self) -> HostPointerWidth { + HostPointerWidth::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P4)) + } + pub const fn read_p5_arch(&self) -> HostArch { + HostArch::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P5)) + } + pub const fn read_p6_os(&self) -> HostOS { + HostOS::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P6)) + } + pub const fn encoded(&self) -> DRHostSignature { + DRHostSignature::new( + self.read_p0_server_version(), + self.read_p1_driver_version(), + self.read_p2_file_specifier_id(), + self.read_p3_endian(), + self.read_p4_pointer_width(), + self.read_p5_arch(), + self.read_p6_os(), + ) + } +} + +/* + Dynamic record (2/2): Runtime signature + --- + - 8B: Dynamic record modify count + - 16B: Host epoch time + - 16B: Host uptime + - 8B: Host name length + - ?B: Host name +*/ + +#[derive(Debug, PartialEq, Clone)] +pub struct DRRuntimeSignature { + rt_signature_fixed: DRRuntimeSignatureFixed, + host_name: Box<[u8]>, +} + +impl DRRuntimeSignature { + pub fn new(fixed: DRRuntimeSignatureFixed, host_name: Box<[u8]>) -> Self { + Self { + rt_signature_fixed: fixed, + host_name, + } + } + pub const fn rt_signature_fixed(&self) -> &DRRuntimeSignatureFixed { + &self.rt_signature_fixed + } + pub fn host_name(&self) -> &[u8] { + self.host_name.as_ref() + } + pub fn into_encoded(self) -> DRRuntimeSignatureRaw { + let len = self.host_name.len(); + DRRuntimeSignatureRaw::new_with_sections( + self.host_name, + self.rt_signature_fixed.encoded(len), + ) + } + pub fn encoded(&self) -> DRRuntimeSignatureRaw { + self.clone().into_encoded() + } +} + +pub struct DRRuntimeSignatureRaw { + rt_signature: DRRuntimeSignatureFixedRaw, + pub(super) host_name: Box<[u8]>, +} + +impl DRRuntimeSignatureRaw { + pub fn new(host_name: Box<[u8]>, modify_count: u64) -> Self { + Self { + rt_signature: DRRuntimeSignatureFixedRaw::new(modify_count, host_name.len()), + host_name, + } + } + pub fn new_with_sections(host_name: Box<[u8]>, fixed: DRRuntimeSignatureFixedRaw) -> Self { + Self { + rt_signature: fixed, + host_name, + } + } + pub fn decode( + data: [u8; sizeof!(DRRuntimeSignatureFixedRaw)], + ) -> Option<(usize, DRRuntimeSignatureFixed)> { + let s = ByteStack::new(data); + let modify_count = u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P0)); + let epoch_time = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P1)); + let uptime = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P2)); + let host_name_length = + u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P3)); + if epoch_time > crate::util::os::get_epoch_time() || host_name_length > usize::MAX as u64 { + // damn, this file is from the future; I WISH EVERYONE HAD NTP SYNC GRRRR + // or, we have a bad host name. like, what? + return None; + } + Some(( + host_name_length as _, + DRRuntimeSignatureFixed::new(modify_count, epoch_time, uptime), + )) + } + pub const fn runtime_signature(&self) -> &DRRuntimeSignatureFixedRaw { + &self.rt_signature + } + pub fn name(&self) -> &[u8] { + &self.host_name + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct DRRuntimeSignatureFixed { + modify_count: u64, + epoch_time: u128, + uptime: u128, +} + +impl DRRuntimeSignatureFixed { + pub const fn new(modify_count: u64, epoch_time: u128, uptime: u128) -> Self { + Self { + modify_count, + epoch_time, + uptime, + } + } + pub const fn modify_count(&self) -> u64 { + self.modify_count + } + pub const fn epoch_time(&self) -> u128 { + self.epoch_time + } + pub const fn uptime(&self) -> u128 { + self.uptime + } + pub fn encoded(&self, host_name_length: usize) -> DRRuntimeSignatureFixedRaw { + DRRuntimeSignatureFixedRaw::new_full( + self.modify_count(), + self.epoch_time(), + self.uptime(), + host_name_length, + ) + } +} + +pub struct DRRuntimeSignatureFixedRaw { + data: ByteStack<48>, +} + +impl DRRuntimeSignatureFixedRaw { + const DRRS_OFFSET_P0: usize = 0; + const DRRS_OFFSET_P1: usize = sizeof!(u64); + const DRRS_OFFSET_P2: usize = Self::DRRS_OFFSET_P1 + sizeof!(u128); + const DRRS_OFFSET_P3: usize = Self::DRRS_OFFSET_P2 + sizeof!(u128); + const _ENSURE: () = assert!(Self::DRRS_OFFSET_P3 == sizeof!(Self) - 8); + pub fn new_full( + modify_count: u64, + epoch_time: u128, + uptime: u128, + host_name_length: usize, + ) -> Self { + let _ = Self::_ENSURE; + let mut data = [0u8; sizeof!(Self)]; + data[0..8].copy_from_slice(&modify_count.to_le_bytes()); + data[8..24].copy_from_slice(&epoch_time.to_le_bytes()); + data[24..40].copy_from_slice(&uptime.to_le_bytes()); + data[40..48].copy_from_slice(&(host_name_length as u64).to_le_bytes()); + Self { + data: ByteStack::new(data), + } + } + pub fn new(modify_count: u64, host_name_length: usize) -> Self { + Self::new_full( + modify_count, + crate::util::os::get_epoch_time(), + crate::util::os::get_uptime(), + host_name_length, + ) + } + pub const fn read_p0_modify_count(&self) -> u64 { + self.data.read_qword(Self::DRRS_OFFSET_P0) + } + pub const fn read_p1_epoch_time(&self) -> u128 { + self.data.read_xmmword(Self::DRRS_OFFSET_P1) + } + pub const fn read_p2_uptime(&self) -> u128 { + self.data.read_xmmword(Self::DRRS_OFFSET_P2) + } + pub const fn read_p3_host_name_length(&self) -> u64 { + self.data.read_qword(Self::DRRS_OFFSET_P3) + } +} diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl/gr.rs similarity index 64% rename from server/src/engine/storage/v1/header_impl.rs rename to server/src/engine/storage/v1/header_impl/gr.rs index fe68c8c8..42fdd4df 100644 --- a/server/src/engine/storage/v1/header_impl.rs +++ b/server/src/engine/storage/v1/header_impl/gr.rs @@ -1,5 +1,5 @@ /* - * Created on Mon May 15 2023 + * Created on Thu May 25 2023 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -24,85 +24,16 @@ * */ -/* - * SDSS Header layout: - * - * +--------------------------------------------------------------+ - * | | - * | STATIC RECORD | - * | 128B | - * +--------------------------------------------------------------+ - * +--------------------------------------------------------------+ - * | | - * | | - * | DYNAMIC RECORD | - * | (256+56+?)B | - * | +--------------------------------------------+ | - * | | | | - * | | METADATA RECORD | | - * | | 256B | | - * | +--------------------------------------------+ | - * | +--------------------------------------------+ | - * | | | | - * | | GENESIS HOST RECORD | | - * | | >56B | | - * | +--------------------------------------------+ | - * | | - * +--------------------------------------------------------------+ - * +--------------------------------------------------------------+ - * | RUNTIME HOST RECORD | - * | >56B | - * +--------------------------------------------------------------+ - * Note: The entire part of the header is little endian encoded -*/ - use crate::engine::{ mem::ByteStack, storage::{ - header::{StaticRecordUV, StaticRecordUVRaw}, + v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode}, versions::{self, DriverVersion, ServerVersion}, }, }; -pub struct StaticRecord { - sr: StaticRecordUV, -} - -impl StaticRecord { - pub const fn new(sr: StaticRecordUV) -> Self { - Self { sr } - } - pub const fn encode(&self) -> StaticRecordRaw { - StaticRecordRaw { - base: self.sr.encode(), - } - } - pub const fn sr(&self) -> &StaticRecordUV { - &self.sr - } -} - -/// Static record -pub struct StaticRecordRaw { - base: StaticRecordUVRaw, -} - -impl StaticRecordRaw { - pub const fn new() -> Self { - Self { - base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION), - } - } - pub const fn empty_buffer() -> [u8; sizeof!(Self)] { - [0u8; sizeof!(Self)] - } - pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option { - StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new) - } -} - /* - Dynamic record (1/2) + Genesis record (1/2) --- Metadata record (8B x 3 + (4B x 2)): +----------+----------+----------+---------+ @@ -112,59 +43,7 @@ impl StaticRecordRaw { 0, 63 */ -/// The file scope -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] -pub enum FileScope { - TransactionLog = 0, - TransactionLogCompacted = 1, -} - -impl FileScope { - pub const fn try_new(id: u64) -> Option { - Some(match id { - 0 => Self::TransactionLog, - 1 => Self::TransactionLogCompacted, - _ => return None, - }) - } - pub const fn new(id: u64) -> Self { - match Self::try_new(id) { - Some(v) => v, - None => panic!("unknown filescope"), - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] -#[repr(u8)] -pub enum FileSpecifier { - GNSTxnLog = 0, -} - -impl FileSpecifier { - pub const fn try_new(v: u32) -> Option { - Some(match v { - 0 => Self::GNSTxnLog, - _ => return None, - }) - } - pub const fn new(v: u32) -> Self { - match Self::try_new(v) { - Some(v) => v, - _ => panic!("unknown filespecifier"), - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct FileSpecifierVersion(u32); -impl FileSpecifierVersion { - pub const fn __new(v: u32) -> Self { - Self(v) - } -} - +#[derive(Debug, PartialEq)] pub struct MetadataRecord { server_version: ServerVersion, driver_version: DriverVersion, @@ -216,7 +95,7 @@ impl MetadataRecord { } pub struct MetadataRecordRaw { - data: ByteStack<32>, + pub(super) data: ByteStack<32>, } impl MetadataRecordRaw { @@ -320,7 +199,7 @@ impl MetadataRecordRaw { } /* - Dynamic Record (2/2) + Genesis Record (2/2) --- Host record (?B; > 56B): - 16B: Host epoch time in nanoseconds @@ -333,30 +212,7 @@ impl MetadataRecordRaw { - ??B: Host name */ -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] -#[repr(u8)] -pub enum HostRunMode { - Dev = 0, - Prod = 1, -} - -impl HostRunMode { - pub const fn try_new_with_val(v: u32) -> Option { - Some(match v { - 0 => Self::Dev, - 1 => Self::Prod, - _ => return None, - }) - } - pub const fn new_with_val(v: u32) -> Self { - match Self::try_new_with_val(v) { - Some(v) => v, - None => panic!("unknown hostrunmode"), - } - } -} - -type HRConstSectionRaw = [u8; 56]; +pub type HRConstSectionRaw = [u8; 56]; #[derive(Debug, PartialEq, Clone)] pub struct HostRecord { @@ -429,8 +285,8 @@ impl HRConstSection { } pub struct HostRecordRaw { - data: ByteStack<{ sizeof!(HRConstSectionRaw) }>, - host_name: Box<[u8]>, + pub(super) data: ByteStack<{ sizeof!(HRConstSectionRaw) }>, + pub(super) host_name: Box<[u8]>, } impl HostRecordRaw { @@ -499,13 +355,18 @@ impl HostRecordRaw { let _ = Self::_ENSURE; let p4_host_name_length = p5_host_name.len(); let mut host_record_fl = [0u8; 56]; - host_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes()); - host_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_le_bytes()); - host_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_le_bytes()); - host_record_fl[36..40] + host_record_fl[Self::HR_OFFSET_P0..Self::HR_OFFSET_P1] + .copy_from_slice(&p0_host_epoch_time.to_le_bytes()); + host_record_fl[Self::HR_OFFSET_P1..Self::HR_OFFSET_P2A] + .copy_from_slice(&p1_host_uptime.to_le_bytes()); + host_record_fl[Self::HR_OFFSET_P2A..Self::HR_OFFSET_P2B] + .copy_from_slice(&p2a_host_setting_version_id.to_le_bytes()); + host_record_fl[Self::HR_OFFSET_P2B..Self::HR_OFFSET_P3] .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_le_bytes()); - host_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes()); - host_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes()); + host_record_fl[Self::HR_OFFSET_P3..Self::HR_OFFSET_P4] + .copy_from_slice(&p3_host_startup_counter.to_le_bytes()); + host_record_fl[Self::HR_OFFSET_P4..] + .copy_from_slice(&(p4_host_name_length as u64).to_le_bytes()); Self { data: ByteStack::new(host_record_fl), host_name: p5_host_name, @@ -554,103 +415,6 @@ impl HostRecordRaw { } } -pub struct SDSSHeader { - sr: StaticRecord, - mdr: MetadataRecord, - hr: HostRecord, -} - -impl SDSSHeader { - pub const fn new(sr: StaticRecord, mdr: MetadataRecord, hr: HostRecord) -> Self { - Self { sr, mdr, hr } - } - pub const fn sr(&self) -> &StaticRecord { - &self.sr - } - pub const fn mdr(&self) -> &MetadataRecord { - &self.mdr - } - pub const fn hr(&self) -> &HostRecord { - &self.hr - } - pub fn encode(&self) -> SDSSHeaderRaw { - SDSSHeaderRaw::new_full(self.sr.encode(), self.mdr.encode(), self.hr.encode()) - } -} - -pub struct SDSSHeaderRaw { - sr: StaticRecordRaw, - dr_0_mdr: MetadataRecordRaw, - dr_1_hr: HostRecordRaw, -} - -impl SDSSHeaderRaw { - pub fn new_full(sr: StaticRecordRaw, mdr: MetadataRecordRaw, hr: HostRecordRaw) -> Self { - Self { - sr, - dr_0_mdr: mdr, - dr_1_hr: hr, - } - } - pub fn new( - sr: StaticRecordRaw, - dr_0_mdr: MetadataRecordRaw, - dr_1_hr_const_section: HRConstSectionRaw, - dr_1_hr_host_name: Box<[u8]>, - ) -> Self { - Self { - sr, - dr_0_mdr, - dr_1_hr: HostRecordRaw { - data: ByteStack::new(dr_1_hr_const_section), - host_name: dr_1_hr_host_name, - }, - } - } - pub fn init( - mdr_file_scope: FileScope, - mdr_file_specifier: FileSpecifier, - mdr_file_specifier_id: FileSpecifierVersion, - hr_host_setting_id: u32, - hr_host_run_mode: HostRunMode, - hr_host_startup_counter: u64, - hr_host_name: Box<[u8]>, - ) -> Self { - Self { - sr: StaticRecordRaw::new(), - dr_0_mdr: MetadataRecordRaw::new( - mdr_file_scope, - mdr_file_specifier, - mdr_file_specifier_id, - ), - dr_1_hr: HostRecordRaw::new_auto( - hr_host_setting_id, - hr_host_run_mode, - hr_host_startup_counter, - hr_host_name, - ), - } - } - pub fn get0_sr(&self) -> &[u8] { - self.sr.base.get_ref() - } - pub fn get1_dr_0_mdr(&self) -> &[u8] { - self.dr_0_mdr.data.slice() - } - pub fn get1_dr_1_hr_0(&self) -> &[u8] { - self.dr_1_hr.data.slice() - } - pub fn get1_dr_1_hr_1(&self) -> &[u8] { - self.dr_1_hr.host_name.as_ref() - } - pub fn calculate_header_size(&self) -> usize { - Self::calculate_fixed_header_size() + self.dr_1_hr.host_name.len() - } - pub const fn calculate_fixed_header_size() -> usize { - sizeof!(StaticRecordRaw) + sizeof!(MetadataRecordRaw) + sizeof!(HRConstSectionRaw) - } -} - #[test] fn test_metadata_record_encode_decode() { let md = MetadataRecordRaw::new( diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs new file mode 100644 index 00000000..2ddf4bba --- /dev/null +++ b/server/src/engine/storage/v1/header_impl/mod.rs @@ -0,0 +1,266 @@ +/* + * Created on Mon May 15 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +/* + * SDSS Header layout: + * + * +--------------------------------------------------------------+ + * | | + * | STATIC RECORD | + * | 128B | + * +--------------------------------------------------------------+ + * +--------------------------------------------------------------+ + * | | + * | | + * | GENESIS RECORD | + * | (256+56+?)B | + * | +--------------------------------------------+ | + * | | | | + * | | METADATA RECORD | | + * | | 256B | | + * | +--------------------------------------------+ | + * | +--------------------------------------------+ | + * | | | | + * | | HOST RECORD | | + * | | >56B | | + * | +--------------------------------------------+ | + * | | + * +--------------------------------------------------------------+ + * +--------------------------------------------------------------+ + * | DYNAMIC RECORD | + * | >56B | + * +--------------------------------------------------------------+ + * Note: The entire part of the header is little endian encoded +*/ + +// (1) sr +mod sr; +// (2) gr +mod gr; +// (3) dr +mod dr; + +use crate::engine::mem::ByteStack; + +/// The file scope +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +pub enum FileScope { + TransactionLog = 0, + TransactionLogCompacted = 1, +} + +impl FileScope { + pub const fn try_new(id: u64) -> Option { + Some(match id { + 0 => Self::TransactionLog, + 1 => Self::TransactionLogCompacted, + _ => return None, + }) + } + pub const fn new(id: u64) -> Self { + match Self::try_new(id) { + Some(v) => v, + None => panic!("unknown filescope"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +#[repr(u8)] +pub enum FileSpecifier { + GNSTxnLog = 0, +} + +impl FileSpecifier { + pub const fn try_new(v: u32) -> Option { + Some(match v { + 0 => Self::GNSTxnLog, + _ => return None, + }) + } + pub const fn new(v: u32) -> Self { + match Self::try_new(v) { + Some(v) => v, + _ => panic!("unknown filespecifier"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct FileSpecifierVersion(u32); +impl FileSpecifierVersion { + pub const fn __new(v: u32) -> Self { + Self(v) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)] +#[repr(u8)] +pub enum HostRunMode { + Dev = 0, + Prod = 1, +} + +impl HostRunMode { + pub const fn try_new_with_val(v: u32) -> Option { + Some(match v { + 0 => Self::Dev, + 1 => Self::Prod, + _ => return None, + }) + } + pub const fn new_with_val(v: u32) -> Self { + match Self::try_new_with_val(v) { + Some(v) => v, + None => panic!("unknown hostrunmode"), + } + } +} + +#[derive(Debug, PartialEq)] +pub struct SDSSHeader { + // static record + sr: sr::StaticRecord, + // genesis record + gr_mdr: gr::MetadataRecord, + gr_hr: gr::HostRecord, + // dynamic record + dr_hs: dr::DRHostSignature, + dr_rs: dr::DRRuntimeSignature, +} + +impl SDSSHeader { + pub const fn new( + sr: sr::StaticRecord, + gr_mdr: gr::MetadataRecord, + gr_hr: gr::HostRecord, + dr_hs: dr::DRHostSignature, + dr_rs: dr::DRRuntimeSignature, + ) -> Self { + Self { + sr, + gr_mdr, + gr_hr, + dr_hs, + dr_rs, + } + } + pub fn encode(&self) -> SDSSHeaderRaw { + SDSSHeaderRaw::new_full( + self.sr.encode(), + self.gr_mdr.encode(), + self.gr_hr.encode(), + self.dr_hs().encoded(), + self.dr_rs().encoded(), + ) + } + pub fn sr(&self) -> &sr::StaticRecord { + &self.sr + } + pub fn gr_mdr(&self) -> &gr::MetadataRecord { + &self.gr_mdr + } + pub fn gr_hr(&self) -> &gr::HostRecord { + &self.gr_hr + } + pub fn dr_hs(&self) -> &dr::DRHostSignature { + &self.dr_hs + } + pub fn dr_rs(&self) -> &dr::DRRuntimeSignature { + &self.dr_rs + } +} + +pub struct SDSSHeaderRaw { + sr: sr::StaticRecordRaw, + gr_0_mdr: gr::MetadataRecordRaw, + gr_1_hr: gr::HostRecordRaw, + dr_0_hs: dr::DRHostSignatureRaw, + dr_1_rs: dr::DRRuntimeSignatureRaw, +} + +impl SDSSHeaderRaw { + pub fn new_full( + sr: sr::StaticRecordRaw, + gr_mdr: gr::MetadataRecordRaw, + gr_hr: gr::HostRecordRaw, + dr_hs: dr::DRHostSignatureRaw, + dr_rs: dr::DRRuntimeSignatureRaw, + ) -> Self { + Self { + sr, + gr_0_mdr: gr_mdr, + gr_1_hr: gr_hr, + dr_0_hs: dr_hs, + dr_1_rs: dr_rs, + } + } + pub fn new( + sr: sr::StaticRecordRaw, + gr_0_mdr: gr::MetadataRecordRaw, + gr_1_hr_const_section: gr::HRConstSectionRaw, + gr_1_hr_host_name: Box<[u8]>, + dr_hs: dr::DRHostSignatureRaw, + dr_rs_const: dr::DRRuntimeSignatureFixedRaw, + dr_rs_host_name: Box<[u8]>, + ) -> Self { + Self { + sr, + gr_0_mdr, + gr_1_hr: gr::HostRecordRaw { + data: ByteStack::new(gr_1_hr_const_section), + host_name: gr_1_hr_host_name, + }, + dr_0_hs: dr_hs, + dr_1_rs: dr::DRRuntimeSignatureRaw::new_with_sections(dr_rs_host_name, dr_rs_const), + } + } + pub fn get0_sr(&self) -> &[u8] { + self.sr.base.get_ref() + } + pub fn get1_dr_0_mdr(&self) -> &[u8] { + self.gr_0_mdr.data.slice() + } + pub fn get1_dr_1_hr_0(&self) -> &[u8] { + self.gr_1_hr.data.slice() + } + pub fn get1_dr_1_hr_1(&self) -> &[u8] { + self.gr_1_hr.host_name.as_ref() + } + pub fn calculate_header_size(&self) -> usize { + Self::calculate_fixed_header_size() + + self.gr_1_hr.host_name.len() + + self.dr_1_rs.host_name.len() + } + pub const fn calculate_fixed_header_size() -> usize { + sizeof!(sr::StaticRecordRaw) + + sizeof!(gr::MetadataRecordRaw) + + sizeof!(gr::HRConstSectionRaw) + + sizeof!(dr::DRHostSignatureRaw) + + sizeof!(dr::DRRuntimeSignatureFixedRaw) + } +} diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs new file mode 100644 index 00000000..a98f2af9 --- /dev/null +++ b/server/src/engine/storage/v1/header_impl/sr.rs @@ -0,0 +1,68 @@ +/* + * Created on Thu May 25 2023 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2023, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +use crate::engine::storage::{ + header::{StaticRecordUV, StaticRecordUVRaw}, + versions, +}; + +#[derive(Debug, PartialEq)] +pub struct StaticRecord { + sr: StaticRecordUV, +} + +impl StaticRecord { + pub const fn new(sr: StaticRecordUV) -> Self { + Self { sr } + } + pub const fn encode(&self) -> StaticRecordRaw { + StaticRecordRaw { + base: self.sr.encode(), + } + } + pub const fn sr(&self) -> &StaticRecordUV { + &self.sr + } +} + +/// Static record +pub struct StaticRecordRaw { + pub(super) base: StaticRecordUVRaw, +} + +impl StaticRecordRaw { + pub const fn new() -> Self { + Self { + base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION), + } + } + pub const fn empty_buffer() -> [u8; sizeof!(Self)] { + [0u8; sizeof!(Self)] + } + pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option { + StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new) + } +} diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs index bc0d88ac..1dc078d2 100644 --- a/server/src/engine/storage/v1/rw.rs +++ b/server/src/engine/storage/v1/rw.rs @@ -23,191 +23,3 @@ * along with this program. If not, see . * */ - -use crate::engine::storage::versions; - -use { - super::header_impl::SDSSHeaderRaw, - crate::{ - engine::storage::v1::header_impl::{ - HostRecord, HostRecordRaw, MetadataRecordRaw, SDSSHeader, StaticRecordRaw, - }, - util::{ByteRepr, NumericRepr}, - IoResult, - }, - std::{ - fs::{self, File, OpenOptions}, - io::{Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write}, - }, -}; - -#[derive(Debug)] -pub enum SDSSError { - SRVersionMismatch, - IoError(IoError), - CorruptedHeaderSR, - CorruptedHeaderMDR, - CorruptedHeaderHR, -} - -impl From for SDSSError { - fn from(e: IoError) -> Self { - Self::IoError(e) - } -} - -pub type SDSSResult = Result; - -/* - Writer interface -*/ - -pub trait RawWriterInterface: Sized { - fn fopen_truncated(fname: &str) -> IoResult; - fn fopen_create(fname: &str) -> IoResult; - fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()>; - fn fsync_all(&mut self) -> IoResult<()>; -} - -impl RawWriterInterface for File { - fn fopen_truncated(fname: &str) -> IoResult { - OpenOptions::new() - .write(true) - .truncate(true) - .create(false) - .open(fname) - } - - fn fopen_create(fname: &str) -> IoResult { - File::create(fname) - } - - fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()> { - Write::write_all(self, bytes) - } - - fn fsync_all(&mut self) -> IoResult<()> { - // FIXME(@ohsayan): too slow? maybe fdatasync only? - File::sync_all(self) - } -} - -/* - Writer -*/ - -pub struct SDSSWriter { - writer: W, -} - -impl SDSSWriter { - pub fn open_create_with_header(file: &str, header: SDSSHeaderRaw) -> IoResult { - let mut w = W::fopen_create(file)?; - w.fwrite_all(header.get0_sr())?; - w.fwrite_all(header.get1_dr_0_mdr())?; - w.fwrite_all(header.get1_dr_1_hr_0())?; - w.fwrite_all(header.get1_dr_1_hr_1())?; - w.fsync_all()?; - Ok(Self { writer: w }) - } - pub fn fsync_write(&mut self, data: &D) -> IoResult<()> { - self.writer.fwrite_all(data.repr())?; - self.writer.fsync_all() - } - pub fn newrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> { - self.fsync_write(num.repr()) - } - pub fn lewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> { - self.fsync_write(num.le().repr()) - } - pub fn bewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> { - self.fsync_write(num.be().repr()) - } -} - -/* - Read interface -*/ - -pub trait RawReaderInterface: Sized { - fn fopen(fname: &str) -> IoResult; - fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()>; - fn fread_to_end(&mut self, buf: &mut Vec) -> IoResult<()>; -} - -impl RawReaderInterface for File { - fn fopen(fname: &str) -> IoResult { - File::open(fname) - } - fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> { - self.read_exact(buf)?; - let _ = self.seek(SeekFrom::Start(buf.len() as _))?; - Ok(()) - } - fn fread_to_end(&mut self, buf: &mut Vec) -> IoResult<()> { - match self.read_to_end(buf) { - Ok(_) => Ok(()), - Err(e) => Err(e), - } - } -} - -pub struct FileBuffered { - cursor: usize, - base: Vec, -} - -impl RawReaderInterface for FileBuffered { - fn fopen(name: &str) -> IoResult { - Ok(Self { - base: fs::read(name)?, - cursor: 0, - }) - } - fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> { - let l = self.base[self.cursor..].len(); - if l >= buf.len() { - self.cursor += buf.len(); - Ok(buf.copy_from_slice(&self.base[self.cursor..])) - } else { - Err(ErrorKind::UnexpectedEof.into()) - } - } - fn fread_to_end(&mut self, buf: &mut Vec) -> IoResult<()> { - buf.extend_from_slice(&self.base[self.cursor..]); - Ok(()) - } -} - -pub struct SDSSReader { - reader: R, - header: SDSSHeader, -} - -impl SDSSReader { - pub fn open(f: &str) -> SDSSResult { - let mut r = R::fopen(f)?; - let mut sr = StaticRecordRaw::empty_buffer(); - let mut mdr = MetadataRecordRaw::empty_buffer(); - let mut hr_0_const = HostRecordRaw::empty_buffer_const_section(); - r.fread_exact_seek(&mut sr)?; - r.fread_exact_seek(&mut mdr)?; - r.fread_exact_seek(&mut hr_0_const)?; - let sr = StaticRecordRaw::decode_from_bytes(sr).ok_or(SDSSError::CorruptedHeaderSR)?; - let mdr = MetadataRecordRaw::decode_from_bytes(mdr).ok_or(SDSSError::CorruptedHeaderMDR)?; - let (hr_const, hostname_len) = HostRecordRaw::decode_from_bytes_const_sec(hr_0_const) - .ok_or(SDSSError::CorruptedHeaderHR)?; - let mut host_name = vec![0u8; hostname_len].into_boxed_slice(); - r.fread_exact_seek(&mut host_name)?; - if (sr.sr().header_version() != versions::v1::V1_HEADER_VERSION) - || (mdr.driver_version() != versions::v1::V1_DRIVER_VERSION) - || (mdr.server_version() != versions::v1::V1_SERVER_VERSION) - { - return Err(SDSSError::SRVersionMismatch); - } - Ok(Self { - reader: r, - header: SDSSHeader::new(sr, mdr, HostRecord::new(hr_const, host_name)), - }) - } -}