Simplify header decode impl and fix MDR endian bug

next
Sayan Nandan 1 year ago
parent 0e77946714
commit beb4d16ab4
No known key found for this signature in database
GPG Key ID: 42EEDF4AE9D96B54

@ -25,6 +25,7 @@
*/
mod astr;
mod stackop;
mod uarray;
mod vinline;
mod word;
@ -32,10 +33,13 @@ mod word;
#[cfg(test)]
mod tests;
// re-exports
pub use astr::AStr;
pub use uarray::UArray;
pub use vinline::VInline;
pub use word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK};
pub use {
astr::AStr,
stackop::ByteStack,
uarray::UArray,
vinline::VInline,
word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK},
};
// imports
use std::alloc::{self, Layout};

@ -0,0 +1,84 @@
/*
* Created on Tue May 23 2023
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
* NoSQL database written by Sayan Nandan ("the Author") with the
* vision to provide flexibility in data modelling without compromising
* on performance, queryability or scalability.
*
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
#[derive(Debug, PartialEq, Clone, Copy)]
pub struct ByteStack<const N: usize> {
array: [u8; N],
}
impl<const N: usize> ByteStack<N> {
#[inline(always)]
pub const fn data_copy(&self) -> [u8; N] {
self.array
}
#[inline(always)]
pub const fn new(array: [u8; N]) -> Self {
Self { array }
}
#[inline(always)]
pub const fn zeroed() -> Self {
Self::new([0u8; N])
}
#[inline(always)]
pub const fn slice(&self) -> &[u8] {
&self.array
}
#[inline(always)]
pub const fn read_byte(&self, position: usize) -> u8 {
self.array[position]
}
#[inline(always)]
pub const fn read_word(&self, position: usize) -> u16 {
unsafe { core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)]) }
}
#[inline(always)]
pub const fn read_dword(&self, position: usize) -> u32 {
unsafe {
core::mem::transmute([
self.read_word(position),
self.read_word(position + sizeof!(u16)),
])
}
}
#[inline(always)]
pub const fn read_qword(&self, position: usize) -> u64 {
unsafe {
core::mem::transmute([
self.read_dword(position),
self.read_dword(position + sizeof!(u32)),
])
}
}
#[inline(always)]
pub const fn read_xmmword(&self, position: usize) -> u128 {
unsafe {
core::mem::transmute([
self.read_qword(position),
self.read_qword(position + sizeof!(u64)),
])
}
}
}

@ -56,7 +56,7 @@
HEADS UP: Static record is always little endian
*/
use super::versions::HeaderVersion;
use {super::versions::HeaderVersion, crate::engine::mem::ByteStack};
/// magic
const SR0_MAGIC: u64 = 0x4F48534159414E21;
@ -264,10 +264,52 @@ impl HostPointerWidth {
}
}
#[derive(Debug, PartialEq)]
pub struct StaticRecordUVDecoded {
header_version: HeaderVersion,
ptr_width: HostPointerWidth,
endian: HostEndian,
arch: HostArch,
os: HostOS,
}
impl StaticRecordUVDecoded {
pub const fn new(
header_version: HeaderVersion,
ptr_width: HostPointerWidth,
endian: HostEndian,
arch: HostArch,
os: HostOS,
) -> Self {
Self {
header_version,
ptr_width,
endian,
arch,
os,
}
}
pub fn header_version(&self) -> HeaderVersion {
self.header_version
}
pub fn ptr_width(&self) -> HostPointerWidth {
self.ptr_width
}
pub fn endian(&self) -> HostEndian {
self.endian
}
pub fn arch(&self) -> HostArch {
self.arch
}
pub fn os(&self) -> HostOS {
self.os
}
}
#[derive(Debug, PartialEq)]
/// The static record
pub struct StaticRecordUV {
data: [u8; 16],
data: ByteStack<16>,
}
impl StaticRecordUV {
@ -299,7 +341,9 @@ impl StaticRecordUV {
data[sizeof!(u64, 2) - 3] = sr3_endian.value_u8();
data[sizeof!(u64, 2) - 2] = sr4_arch.value_u8();
data[sizeof!(u64, 2) - 1] = sr5_os.value_u8();
Self { data }
Self {
data: ByteStack::new(data),
}
}
#[inline(always)]
pub const fn create(sr1_version: HeaderVersion) -> Self {
@ -308,19 +352,22 @@ impl StaticRecordUV {
/// Decode and validate a SR
///
/// WARNING: NOT CONTEXTUAL! VALIDATE YOUR OWN STUFF!
pub fn decode(data: [u8; 16]) -> Option<Self> {
pub fn decode(data: [u8; 16]) -> Option<StaticRecordUVDecoded> {
let _ = Self::_ENSURE;
let slf = Self { data };
let slf = Self {
data: ByteStack::new(data),
};
// p0: magic; the magic HAS to be the same
if u64::from_le(slf.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
if u64::from_le(slf.data.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
return None;
}
let sr2_ptr = HostPointerWidth::try_new_with_val(slf.read_byte(Self::OFFSET_P2))?; // p2: ptr width
let sr3_endian = HostEndian::try_new_with_val(slf.read_byte(Self::OFFSET_P3))?; // p3: endian
let sr4_arch = HostArch::try_new_with_val(slf.read_byte(Self::OFFSET_P4))?; // p4: arch
let sr5_os = HostOS::try_new_with_val(slf.read_byte(Self::OFFSET_P5))?; // p5: os
Some(Self::new(
HeaderVersion::__new(u32::from_le(slf.read_dword(Self::OFFSET_P1))),
let sr1_header_version = HeaderVersion::__new(slf.data.read_dword(Self::OFFSET_P1));
let sr2_ptr = HostPointerWidth::try_new_with_val(slf.data.read_byte(Self::OFFSET_P2))?; // p2: ptr width
let sr3_endian = HostEndian::try_new_with_val(slf.data.read_byte(Self::OFFSET_P3))?; // p3: endian
let sr4_arch = HostArch::try_new_with_val(slf.data.read_byte(Self::OFFSET_P4))?; // p4: arch
let sr5_os = HostOS::try_new_with_val(slf.data.read_byte(Self::OFFSET_P5))?; // p5: os
Some(StaticRecordUVDecoded::new(
sr1_header_version,
sr2_ptr,
sr3_endian,
sr4_arch,
@ -331,30 +378,37 @@ impl StaticRecordUV {
impl StaticRecordUV {
pub const fn get_ref(&self) -> &[u8] {
&self.data
self.data.slice()
}
pub const fn read_p0_magic(&self) -> u64 {
self.read_qword(Self::OFFSET_P0)
self.data.read_qword(Self::OFFSET_P0)
}
pub const fn read_p1_header_version(&self) -> HeaderVersion {
HeaderVersion::__new(self.read_dword(Self::OFFSET_P1))
HeaderVersion::__new(self.data.read_dword(Self::OFFSET_P1))
}
pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
HostPointerWidth::new_with_val(self.read_byte(Self::OFFSET_P2))
HostPointerWidth::new_with_val(self.data.read_byte(Self::OFFSET_P2))
}
pub const fn read_p3_endian(&self) -> HostEndian {
HostEndian::new_with_val(self.read_byte(Self::OFFSET_P3))
HostEndian::new_with_val(self.data.read_byte(Self::OFFSET_P3))
}
pub const fn read_p4_arch(&self) -> HostArch {
HostArch::new_with_val(self.read_byte(Self::OFFSET_P4))
HostArch::new_with_val(self.data.read_byte(Self::OFFSET_P4))
}
pub const fn read_p5_os(&self) -> HostOS {
HostOS::new_with_val(self.read_byte(Self::OFFSET_P5))
HostOS::new_with_val(self.data.read_byte(Self::OFFSET_P5))
}
pub const fn decoded(&self) -> StaticRecordUVDecoded {
StaticRecordUVDecoded::new(
self.read_p1_header_version(),
self.read_p2_ptr_width(),
self.read_p3_endian(),
self.read_p4_arch(),
self.read_p5_os(),
)
}
}
impl_stack_read_primitives!(unsafe impl for StaticRecordUV {});
#[test]
fn test_static_record() {
let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
@ -372,6 +426,6 @@ fn test_static_record() {
#[test]
fn test_static_record_encode_decode() {
let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
let static_record_decoded = StaticRecordUV::decode(static_record.data).unwrap();
assert_eq!(static_record, static_record_decoded);
let static_record_decoded = StaticRecordUV::decode(static_record.data.data_copy()).unwrap();
assert_eq!(static_record.decoded(), static_record_decoded);
}

@ -1,65 +0,0 @@
/*
* Created on Thu May 18 2023
*
* This file is a part of Skytable
* Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
* NoSQL database written by Sayan Nandan ("the Author") with the
* vision to provide flexibility in data modelling without compromising
* on performance, queryability or scalability.
*
* Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
// HACK(@ohsayan): until const traits are stable, this is the silly stuff we have to resort to
macro_rules! impl_stack_read_primitives {
(unsafe impl for $ty:ty {}) => {
impl $ty {
const fn read_byte(&self, position: usize) -> u8 {
self.data[position]
}
const fn read_word(&self, position: usize) -> u16 {
unsafe {
core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)])
}
}
const fn read_dword(&self, position: usize) -> u32 {
unsafe {
core::mem::transmute([
self.read_word(position),
self.read_word(position + sizeof!(u16)),
])
}
}
const fn read_qword(&self, position: usize) -> u64 {
unsafe {
core::mem::transmute([
self.read_dword(position),
self.read_dword(position + sizeof!(u32)),
])
}
}
const fn read_xmmword(&self, position: usize) -> u128 {
unsafe {
core::mem::transmute([
self.read_qword(position),
self.read_qword(position + sizeof!(u64)),
])
}
}
}
};
}

@ -26,8 +26,6 @@
//! Implementations of the Skytable Disk Storage Subsystem (SDSS)
#[macro_use]
mod macros;
mod header;
mod versions;
// impls

@ -52,9 +52,12 @@
* Note: The entire part of the header is little endian encoded
*/
use crate::engine::storage::{
header::StaticRecordUV,
versions::{self, DriverVersion, ServerVersion},
use crate::engine::{
mem::ByteStack,
storage::{
header::StaticRecordUV,
versions::{self, DriverVersion, ServerVersion},
},
};
/// Static record
@ -135,25 +138,27 @@ impl FileSpecifierVersion {
}
pub struct MetadataRecord {
data: [u8; 32],
data: ByteStack<32>,
}
impl_stack_read_primitives!(unsafe impl for MetadataRecord {});
impl MetadataRecord {
/// Decodes a given metadata record, validating all data for correctness.
///
/// WARNING: That means you need to do contextual validation! This function is not aware of any context
pub fn decode(data: [u8; 32]) -> Option<Self> {
let slf = Self { data };
let slf = Self {
data: ByteStack::new(data),
};
let server_version =
ServerVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P0)));
ServerVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P0)));
let driver_version =
DriverVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P1)));
let file_scope = FileScope::try_new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P2)))?;
let file_spec = FileSpecifier::try_new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P3)))?;
DriverVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P1)));
let file_scope =
FileScope::try_new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P2)))?;
let file_spec =
FileSpecifier::try_new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P3)))?;
let file_spec_id =
FileSpecifierVersion::__new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P4)));
FileSpecifierVersion::__new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P4)));
Some(Self::new_full(
server_version,
driver_version,
@ -186,8 +191,12 @@ impl MetadataRecord {
let driver_version = driver_version.little_endian();
let file_scope = scope.value_qword().to_le_bytes();
// specifier + specifier ID
let file_specifier_and_id: u64 =
unsafe { core::mem::transmute([specifier.value_u8() as u32, specifier_id.0]) };
let file_specifier_and_id: u64 = unsafe {
core::mem::transmute([
(specifier.value_u8() as u32).to_le(),
specifier_id.0.to_le(),
])
};
let file_specifier_and_id = file_specifier_and_id.to_le_bytes();
while i < sizeof!(u64) {
ret[i] = server_version[i];
@ -196,7 +205,9 @@ impl MetadataRecord {
ret[i + sizeof!(u64, 3)] = file_specifier_and_id[i];
i += 1;
}
Self { data: ret }
Self {
data: ByteStack::new(ret),
}
}
pub const fn new(
scope: FileScope,
@ -215,19 +226,19 @@ impl MetadataRecord {
impl MetadataRecord {
pub const fn read_p0_server_version(&self) -> ServerVersion {
ServerVersion::__new(self.read_qword(Self::MDR_OFFSET_P0))
ServerVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P0))
}
pub const fn read_p1_driver_version(&self) -> DriverVersion {
DriverVersion::__new(self.read_qword(Self::MDR_OFFSET_P1))
DriverVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P1))
}
pub const fn read_p2_file_scope(&self) -> FileScope {
FileScope::new(self.read_qword(Self::MDR_OFFSET_P2))
FileScope::new(self.data.read_qword(Self::MDR_OFFSET_P2))
}
pub const fn read_p3_file_spec(&self) -> FileSpecifier {
FileSpecifier::new(self.read_dword(Self::MDR_OFFSET_P3))
FileSpecifier::new(self.data.read_dword(Self::MDR_OFFSET_P3))
}
pub const fn read_p4_file_spec_version(&self) -> FileSpecifierVersion {
FileSpecifierVersion(self.read_dword(Self::MDR_OFFSET_P4))
FileSpecifierVersion(self.data.read_dword(Self::MDR_OFFSET_P4))
}
}
@ -270,13 +281,35 @@ impl HostRunMode {
type VHRConstSection = [u8; 56];
impl_stack_read_primitives!(unsafe impl for VariableHostRecord {});
pub struct VariableHostRecord {
data: VHRConstSection,
data: ByteStack<{ sizeof!(VHRConstSection) }>,
host_name: Box<[u8]>,
}
impl VariableHostRecord {
/// Decodes and validates the [`VHRConstSection`] of a [`VariableHostRecord`]. Use the returned result to construct this
pub fn decode(
data: VHRConstSection,
) -> Option<(ByteStack<{ sizeof!(VHRConstSection) }>, usize)> {
let s = ByteStack::new(data);
let host_epoch_time = s.read_xmmword(Self::VHR_OFFSET_P0);
if host_epoch_time > crate::util::os::get_epoch_time() {
// and what? we have a file from the future. Einstein says hi. (ok, maybe the host time is incorrect)
return None;
}
let _host_uptime = s.read_xmmword(Self::VHR_OFFSET_P1);
let _host_setting_version_id = s.read_dword(Self::VHR_OFFSET_P2A);
let _host_setting_run_mode = s.read_dword(Self::VHR_OFFSET_P2B);
let _host_startup_counter = s.read_qword(Self::VHR_OFFSET_P3);
let host_name_length = s.read_qword(Self::VHR_OFFSET_P4);
if host_name_length as usize > usize::MAX {
// too large for us to load. per DNS standards this shouldn't be more than 255 but who knows, some people like it wild
return None;
}
Some((s, host_name_length as usize))
}
}
impl VariableHostRecord {
const VHR_OFFSET_P0: usize = 0;
const VHR_OFFSET_P1: usize = sizeof!(u128);
@ -304,7 +337,7 @@ impl VariableHostRecord {
variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
Self {
data: variable_record_fl,
data: ByteStack::new(variable_record_fl),
host_name: p5_host_name,
}
}
@ -329,22 +362,22 @@ impl VariableHostRecord {
impl VariableHostRecord {
pub const fn read_p0_epoch_time(&self) -> u128 {
self.read_xmmword(Self::VHR_OFFSET_P0)
self.data.read_xmmword(Self::VHR_OFFSET_P0)
}
pub const fn read_p1_uptime(&self) -> u128 {
self.read_xmmword(Self::VHR_OFFSET_P1)
self.data.read_xmmword(Self::VHR_OFFSET_P1)
}
pub const fn read_p2a_setting_version_id(&self) -> u32 {
self.read_dword(Self::VHR_OFFSET_P2A)
self.data.read_dword(Self::VHR_OFFSET_P2A)
}
pub const fn read_p2b_run_mode(&self) -> HostRunMode {
HostRunMode::new_with_val(self.read_dword(Self::VHR_OFFSET_P2B) as u8)
HostRunMode::new_with_val(self.data.read_dword(Self::VHR_OFFSET_P2B) as u8)
}
pub const fn read_p3_startup_counter(&self) -> u64 {
self.read_qword(Self::VHR_OFFSET_P3)
self.data.read_qword(Self::VHR_OFFSET_P3)
}
pub const fn read_p4_host_name_length(&self) -> u64 {
self.read_qword(Self::VHR_OFFSET_P4)
self.data.read_qword(Self::VHR_OFFSET_P4)
}
pub fn read_p5_host_name(&self) -> &[u8] {
&self.host_name
@ -368,7 +401,7 @@ impl SDSSHeader {
sr,
dr_0_mdr,
dr_1_vhr: VariableHostRecord {
data: dr_1_vhr_const_section,
data: ByteStack::new(dr_1_vhr_const_section),
host_name: dr_1_vhr_host_name,
},
}
@ -401,10 +434,10 @@ impl SDSSHeader {
self.sr.base.get_ref()
}
pub fn get1_dr_0_mdr(&self) -> &[u8] {
&self.dr_0_mdr.data
self.dr_0_mdr.data.slice()
}
pub fn get1_dr_1_vhr_0(&self) -> &[u8] {
&self.dr_1_vhr.data
self.dr_1_vhr.data.slice()
}
pub fn get1_dr_1_vhr_1(&self) -> &[u8] {
self.dr_1_vhr.host_name.as_ref()

Loading…
Cancel
Save