From 710fd79e647ada5bd0245f3512f1a9053868eebe Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Mon, 31 Jul 2023 07:52:33 +0000 Subject: [PATCH] Add methods to verify header data --- .../src/engine/storage/v1/header_impl/dr.rs | 29 +++++++++-- .../src/engine/storage/v1/header_impl/gr.rs | 41 +++++++++++++-- .../src/engine/storage/v1/header_impl/mod.rs | 40 +++++++++++--- .../src/engine/storage/v1/header_impl/sr.rs | 17 +++++- server/src/engine/storage/v1/mod.rs | 5 ++ server/src/engine/storage/v1/rw.rs | 13 +++-- server/src/engine/storage/v1/tests.rs | 6 +-- server/src/engine/storage/v1/txn.rs | 52 +++++++++++++++---- 8 files changed, 171 insertions(+), 32 deletions(-) diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs index d7d7647e..f80eea2b 100644 --- a/server/src/engine/storage/v1/header_impl/dr.rs +++ b/server/src/engine/storage/v1/header_impl/dr.rs @@ -29,7 +29,7 @@ use crate::{ mem::ByteStack, storage::{ header::{HostArch, HostEndian, HostOS, HostPointerWidth}, - v1::header_impl::FileSpecifierVersion, + v1::{header_impl::FileSpecifierVersion, SDSSError, SDSSResult}, versions::{self, DriverVersion, ServerVersion}, }, }, @@ -59,11 +59,26 @@ pub struct DRHostSignature { os: HostOS, } +impl DRHostSignature { + pub fn verify(&self, expected_file_specifier_version: FileSpecifierVersion) -> SDSSResult<()> { + if self.server_version() != versions::v1::V1_SERVER_VERSION { + return Err(SDSSError::ServerVersionMismatch); + } + if self.driver_version() != versions::v1::V1_DRIVER_VERSION { + return Err(SDSSError::DriverVersionMismatch); + } + if self.file_specifier_version() != expected_file_specifier_version { + return Err(SDSSError::HeaderDataMismatch); + } + Ok(()) + } +} + impl DRHostSignature { /// Decode the [`DRHostSignature`] from the given bytes /// /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!** - pub fn decode(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option { + pub fn decode_noverify(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option { let ns = ByteStack::new(bytes); let server_version = ServerVersion::__new(u64::from_le( ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P0), @@ -268,7 +283,15 @@ pub struct DRRuntimeSignature { } impl DRRuntimeSignature { - pub fn decode(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option { + pub fn verify(&self) -> SDSSResult<()> { + let et = util::os::get_epoch_time(); + if self.epoch_time() > et || self.host_uptime() > et { + // a file from the future? + return Err(SDSSError::TimeConflict); + } + Ok(()) + } + pub fn decode_noverify(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option { let bytes = ByteStack::new(bytes); // check let modify_count = u64::from_le(bytes.read_qword(DRRuntimeSignatureRaw::DRRS_OFFSET_P0)); diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs index 072574e5..1565dbfd 100644 --- a/server/src/engine/storage/v1/header_impl/gr.rs +++ b/server/src/engine/storage/v1/header_impl/gr.rs @@ -28,7 +28,10 @@ use crate::{ engine::{ mem::ByteStack, storage::{ - v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode}, + v1::{ + header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode}, + SDSSError, SDSSResult, + }, versions::{self, DriverVersion, ServerVersion}, }, }, @@ -55,6 +58,30 @@ pub struct GRMetadataRecord { file_spec_id: FileSpecifierVersion, } +impl GRMetadataRecord { + pub fn verify( + &self, + expected_file_scope: FileScope, + expected_file_specifier: FileSpecifier, + expected_file_specifier_version: FileSpecifierVersion, + ) -> SDSSResult<()> { + if self.server_version() != versions::v1::V1_SERVER_VERSION { + return Err(SDSSError::ServerVersionMismatch); + } + if self.driver_version() != versions::v1::V1_DRIVER_VERSION { + return Err(SDSSError::DriverVersionMismatch); + } + let okay = self.file_scope() == expected_file_scope + && self.file_spec() == expected_file_specifier + && self.file_spec_id() == expected_file_specifier_version; + if okay { + Ok(()) + } else { + Err(SDSSError::HeaderDataMismatch) + } + } +} + impl GRMetadataRecord { pub const fn new( server_version: ServerVersion, @@ -106,7 +133,7 @@ impl GRMetadataRecordRaw { /// Decodes a given metadata record, validating all data for correctness. /// /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!** - pub fn decode(data: [u8; 32]) -> Option { + pub fn decode_noverify(data: [u8; 32]) -> Option { let data = ByteStack::new(data); let server_version = ServerVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P0))); @@ -242,7 +269,15 @@ pub struct GRHostRecord { } impl GRHostRecord { - pub fn decode(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option { + /// Verified: N/A + /// To verify: N/A + pub fn verify(&self) -> SDSSResult<()> { + Ok(()) + } +} + +impl GRHostRecord { + pub fn decode_noverify(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option { let ns = ByteStack::new(bytes); let epoch_time = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P0)); let uptime = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P1)); diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs index 6b58a93c..75e6bb9e 100644 --- a/server/src/engine/storage/v1/header_impl/mod.rs +++ b/server/src/engine/storage/v1/header_impl/mod.rs @@ -58,6 +58,8 @@ use crate::util::copy_slice_to_array as cp; +use super::SDSSResult; + // (1) sr mod sr; // (2) gr @@ -153,6 +155,26 @@ pub struct SDSSHeader { dr_rs: dr::DRRuntimeSignature, } +impl SDSSHeader { + pub fn verify( + &self, + expected_file_scope: FileScope, + expected_file_specifier: FileSpecifier, + expected_file_specifier_version: FileSpecifierVersion, + ) -> SDSSResult<()> { + self.sr().verify()?; + self.gr_mdr().verify( + expected_file_scope, + expected_file_specifier, + expected_file_specifier_version, + )?; + self.gr_hr().verify()?; + self.dr_hs().verify(expected_file_specifier_version)?; + self.dr_rs().verify()?; + Ok(()) + } +} + impl SDSSHeader { pub const fn new( sr: sr::StaticRecord, @@ -290,13 +312,17 @@ impl SDSSHeaderRaw { data } /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!** - pub fn decode(slice: [u8; Self::header_size()]) -> Option { - let sr = sr::StaticRecordRaw::decode(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?; - let gr_mdr = - gr::GRMetadataRecordRaw::decode(cp(&slice[Self::OFFSET_SR1..Self::OFFSET_SR2]))?; - let gr_hr = gr::GRHostRecord::decode(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?; - let dr_sig = dr::DRHostSignature::decode(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?; - let dr_rt = dr::DRRuntimeSignature::decode(cp(&slice[Self::OFFSET_SR4..]))?; + pub fn decode_noverify(slice: [u8; Self::header_size()]) -> Option { + let sr = + sr::StaticRecordRaw::decode_noverify(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?; + let gr_mdr = gr::GRMetadataRecordRaw::decode_noverify(cp( + &slice[Self::OFFSET_SR1..Self::OFFSET_SR2] + ))?; + let gr_hr = + gr::GRHostRecord::decode_noverify(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?; + let dr_sig = + dr::DRHostSignature::decode_noverify(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?; + let dr_rt = dr::DRRuntimeSignature::decode_noverify(cp(&slice[Self::OFFSET_SR4..]))?; Some(SDSSHeader::new(sr, gr_mdr, gr_hr, dr_sig, dr_rt)) } } diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs index 46645c21..3dcdffb2 100644 --- a/server/src/engine/storage/v1/header_impl/sr.rs +++ b/server/src/engine/storage/v1/header_impl/sr.rs @@ -26,6 +26,7 @@ use crate::engine::storage::{ header::{StaticRecordUV, StaticRecordUVRaw}, + v1::{SDSSError, SDSSResult}, versions, }; @@ -34,6 +35,20 @@ pub struct StaticRecord { sr: StaticRecordUV, } +impl StaticRecord { + /// Verified: + /// - header version + /// + /// Need to verify: N/A + pub fn verify(&self) -> SDSSResult<()> { + if self.sr().header_version() == versions::v1::V1_HEADER_VERSION { + Ok(()) + } else { + return Err(SDSSError::HeaderVersionMismatch); + } + } +} + impl StaticRecord { pub const fn new(sr: StaticRecordUV) -> Self { Self { sr } @@ -64,7 +79,7 @@ impl StaticRecordRaw { pub const fn empty_buffer() -> [u8; sizeof!(Self)] { [0u8; sizeof!(Self)] } - pub fn decode(buf: [u8; sizeof!(Self)]) -> Option { + pub fn decode_noverify(buf: [u8; sizeof!(Self)]) -> Option { StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new) } } diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs index d4b14534..2e9931a7 100644 --- a/server/src/engine/storage/v1/mod.rs +++ b/server/src/engine/storage/v1/mod.rs @@ -59,6 +59,11 @@ pub enum SDSSError { CorruptedHeader, TransactionLogEntryCorrupted, TransactionLogCorrupted, + HeaderVersionMismatch, + DriverVersionMismatch, + ServerVersionMismatch, + HeaderDataMismatch, + TimeConflict, } impl SDSSError { diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs index 954bab6d..2c15b3d8 100644 --- a/server/src/engine/storage/v1/rw.rs +++ b/server/src/engine/storage/v1/rw.rs @@ -56,7 +56,7 @@ pub trait RawFileIOInterface: Sized { fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>; fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>; fn fsync_all(&mut self) -> SDSSResult<()>; - fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()>; + fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()>; fn flen(&self) -> SDSSResult; } @@ -89,8 +89,8 @@ impl RawFileIOInterface for File { fn flen(&self) -> SDSSResult { Ok(self.metadata()?.len()) } - fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> { - self.seek(SeekFrom::Start(by as _))?; + fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> { + self.seek(SeekFrom::Start(by))?; Ok(()) } } @@ -132,7 +132,10 @@ impl SDSSFileIO { // this is an existing file. decoded the header let mut header_raw = [0u8; SDSSHeaderRaw::header_size()]; f.fread_exact(&mut header_raw)?; - let header = SDSSHeaderRaw::decode(header_raw).ok_or(SDSSError::CorruptedHeader)?; + let header = + SDSSHeaderRaw::decode_noverify(header_raw).ok_or(SDSSError::CorruptedHeader)?; + // now validate the header + header.verify(file_scope, file_specifier, file_specifier_version)?; // since we updated this file, let us update the header let mut new_header = header.clone(); new_header.dr_rs_mut().bump_modify_count(); @@ -165,7 +168,7 @@ impl SDSSFileIO { pub fn file_length(&self) -> SDSSResult { self.f.flen() } - pub fn seek_ahead(&mut self, by: usize) -> SDSSResult<()> { + pub fn seek_ahead(&mut self, by: u64) -> SDSSResult<()> { self.f.fseek_ahead(by) } } diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs index e1f2bba0..ce0e15e4 100644 --- a/server/src/engine/storage/v1/tests.rs +++ b/server/src/engine/storage/v1/tests.rs @@ -75,8 +75,8 @@ impl VirtualFile { fn r(data: Vec) -> Self { Self::new(true, false, data) } - fn seek_forward(&mut self, by: usize) { - self.pos += by as u64; + fn seek_forward(&mut self, by: u64) { + self.pos += by; assert!(self.pos <= self.data.len() as u64); } fn data(&self) -> &[u8] { @@ -123,7 +123,7 @@ impl RawFileIOInterface for VirtualFileInterface { fn flen(&self) -> SDSSResult { vfs(&self.0, |f| Ok(f.data.len() as _)) } - fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> { + fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> { vfs(&self.0, |f| { f.seek_forward(by); Ok(()) diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs index e1b90fd7..d9410551 100644 --- a/server/src/engine/storage/v1/txn.rs +++ b/server/src/engine/storage/v1/txn.rs @@ -38,15 +38,45 @@ use { super::{ - rw::{RawFileIOInterface, SDSSFileIO}, + header_impl::{FileSpecifierVersion, HostRunMode}, + rw::{FileOpen, RawFileIOInterface, SDSSFileIO}, SDSSError, SDSSResult, }, - crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy}, + crate::{ + engine::storage::v1::header_impl::{FileScope, FileSpecifier}, + util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy}, + }, std::marker::PhantomData, }; const CRC: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); +pub fn open_log( + log_file_name: &str, + log_kind: FileSpecifier, + log_kind_version: FileSpecifierVersion, + host_setting_version: u32, + host_run_mode: HostRunMode, + host_startup_counter: u64, + gs: &TA::GlobalState, +) -> SDSSResult> { + let f = SDSSFileIO::::open_or_create_perm_rw( + log_file_name, + FileScope::TransactionLog, + log_kind, + log_kind_version, + host_setting_version, + host_run_mode, + host_startup_counter, + )?; + let file = match f { + FileOpen::Created(f) => return TransactionLogWriter::new(f, 0, 0), + FileOpen::Existing(file, _) => file, + }; + let (file, size, last_txn) = TransactionLogReader::::scroll(file, gs)?; + TransactionLogWriter::new(file, size, last_txn) +} + /// The transaction adapter pub trait TransactionLogAdapter { /// The transaction event @@ -143,6 +173,7 @@ impl TxnLogEntryMetadata { #[derive(Debug)] pub struct TransactionLogReader { log_file: SDSSFileIO, + log_size: u64, evid: u64, closed: bool, remaining_bytes: u64, @@ -154,6 +185,7 @@ impl TransactionLogReader TransactionLogReader, gs: &TA::GlobalState) -> SDSSResult> { + /// Read and apply all events in the given log file to the global state, returning the + /// (open file, log size, last event ID) + pub fn scroll( + file: SDSSFileIO, + gs: &TA::GlobalState, + ) -> SDSSResult<(SDSSFileIO, u64, u64)> { let mut slf = Self::new(file)?; while !slf.end_of_file() { slf.rapply_next_event(gs)?; } - Ok(slf.log_file) + Ok((slf.log_file, slf.log_size, slf.evid)) } } @@ -244,11 +280,7 @@ pub struct TransactionLogWriter { } impl TransactionLogWriter { - pub fn new( - mut log_file: SDSSFileIO, - last_txn_id: u64, - last_size: usize, - ) -> SDSSResult { + pub fn new(mut log_file: SDSSFileIO, last_size: u64, last_txn_id: u64) -> SDSSResult { log_file.seek_ahead(last_size)?; Ok(Self { log_file,