@ -184,217 +184,317 @@ unsafe fn raw_byte_repr<'a, T: 'a>(len: &'a T) -> &'a [u8] {
} }
} }
/// Serialize a map into a _writable_ thing
mod se {
pub fn serialize_map ( map : & Coremap < Data , Data > ) -> Result < Vec < u8 > , std ::io ::Error > { use super ::* ;
/* use crate ::coredb ::memstore ::Keyspace ;
[ LEN :8 B ] [ KLEN :8 B | VLEN :8 B ] [ K ] [ V ] [ KLEN :8 B ] [ VLEN :8 B ] .. . /// Serialize a map into a _writable_ thing
* / pub fn serialize_map (
// write the len header first
map : & Coremap < Data , Data > ,
let mut w = Vec ::with_capacity ( 128 ) ; model_code : u8 ,
self ::raw_serialize_map ( map , & mut w ) ? ; ) -> Result < Vec < u8 > , std ::io ::Error > {
Ok ( w ) /*
} [ 1 B : Model Mark ] [ LEN :8 B ] [ KLEN :8 B | VLEN :8 B ] [ K ] [ V ] [ KLEN :8 B ] [ VLEN :8 B ] .. .
* /
// write the len header first
let mut w = Vec ::with_capacity ( 128 ) ;
self ::raw_serialize_map ( map , & mut w , model_code ) ? ;
Ok ( w )
}
/// Serialize a map and write it to a provided buffer
/// Serialize a map and write it to a provided buffer
pub fn raw_serialize_map < W : Write > ( map : & Coremap < Data , Data > , w : & mut W ) -> std ::io ::Result < ( ) > { pub fn raw_serialize_map < W : Write > (
unsafe { map : & Coremap < Data , Data > ,
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( map . len ( ) ) ) ) ? ; w : & mut W ,
// now the keys and values
model_code : u8 ,
for kv in map . iter ( ) { ) -> std ::io ::Result < ( ) > {
let ( k , v ) = ( kv . key ( ) , kv . value ( ) ) ; unsafe {
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( k . len ( ) ) ) ) ? ; w . write_all ( raw_byte_repr ( & model_code ) ) ? ;
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( v . len ( ) ) ) ) ? ; w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( map . len ( ) ) ) ) ? ;
w . write_all ( k ) ? ; // now the keys and values
w . write_all ( v ) ? ; for kv in map . iter ( ) {
let ( k , v ) = ( kv . key ( ) , kv . value ( ) ) ;
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( k . len ( ) ) ) ) ? ;
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( v . len ( ) ) ) ) ? ;
w . write_all ( k ) ? ;
w . write_all ( v ) ? ;
}
} }
Ok ( ( ) )
} }
Ok ( ( ) )
}
/// Serialize a set and write it to a provided buffer
/// Serialize a set and write it to a provided buffer
pub fn raw_serialize_set < W , K , V > ( map : & Coremap < K , V > , w : & mut W ) -> std ::io ::Result < ( ) > pub fn raw_serialize_set < W , K , V > ( map : & Coremap < K , V > , w : & mut W ) -> std ::io ::Result < ( ) >
where where
W : Write , W : Write ,
K : Eq + Hash + AsRef < [ u8 ] > , K : Eq + Hash + AsRef < [ u8 ] > ,
{ {
unsafe { unsafe {
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( map . len ( ) ) ) ) ? ; w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( map . len ( ) ) ) ) ? ;
// now the keys and values
// now the keys and values
for kv in map . iter ( ) { for kv in map . iter ( ) {
let key = kv . key ( ) . as_ref ( ) ; let key = kv . key ( ) . as_ref ( ) ;
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( key . len ( ) ) ) ) ? ; w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( key . len ( ) ) ) ) ? ;
w . write_all ( key ) ? ; w . write_all ( key ) ? ;
}
} }
Ok ( ( ) )
} }
Ok ( ( ) )
}
pub trait DeserializeFrom { /// Generate a partition map for the given keyspace
fn is_expected_len ( clen : usize ) -> bool ; /// ```text
fn from_slice ( slice : & [ u8 ] ) -> Self ; /// [8B: EXTENT]([8B: LEN][?B: PARTITION ID][1B: Storage type])*
/// ```
pub fn raw_serialize_partmap < W : Write > ( w : & mut W , keyspace : & Keyspace ) -> std ::io ::Result < ( ) > {
unsafe {
// extent
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( keyspace
. tables
. len ( ) ) ) ) ? ;
for table in keyspace . tables . iter ( ) {
// partition ID
w . write_all ( raw_byte_repr ( & to_64bit_little_endian ! ( table . key ( ) . len ( ) ) ) ) ? ;
// now storage type
w . write_all ( raw_byte_repr ( & table . storage_type ( ) ) ) ? ;
}
}
Ok ( ( ) )
}
} }
impl < const N : usize > DeserializeFrom for Array < u8 , N > { mod de {
fn is_expected_len ( clen : usize ) -> bool { use super ::* ;
clen < = N use std ::collections ::HashMap ;
pub trait DeserializeFrom {
fn is_expected_len ( clen : usize ) -> bool ;
fn from_slice ( slice : & [ u8 ] ) -> Self ;
} }
fn from_slice ( slice : & [ u8 ] ) -> Self {
unsafe { Self ::from_slice ( slice ) } impl < const N : usize > DeserializeFrom for Array < u8 , N > {
fn is_expected_len ( clen : usize ) -> bool {
clen < = N
}
fn from_slice ( slice : & [ u8 ] ) -> Self {
unsafe { Self ::from_slice ( slice ) }
}
} }
}
/// Deserialize a set to a custom type
/// Deserialize a set to a custom type
pub fn deserialize_set_ctype < T > ( data : & [ u8 ] ) -> Option < HashSet < T > > pub fn deserialize_set_ctype < T > ( data : & [ u8 ] ) -> Option < HashSet < T > >
where where
T : DeserializeFrom + Eq + Hash , T : DeserializeFrom + Eq + Hash ,
{ {
// First read the length header
// First read the length header
if data . len ( ) < 8 { if data . len ( ) < 8 {
// so the file doesn't even have the length header? noice, just return
// so the file doesn't even have the length header? noice, just return
None None
} else { } else {
unsafe { unsafe {
// so we have 8B. Just unsafe access and transmute it
// so we have 8B. Just unsafe access and transmute it
let len = transmute_len ( data . as_ptr ( ) ) ; let len = transmute_len ( data . as_ptr ( ) ) ;
let mut set = HashSet ::with_capacity ( len ) ; let mut set = HashSet ::with_capacity ( len ) ;
// this is what we have left: [KLEN:8B]*
// this is what we have left: [KLEN:8B]*
// move 8 bytes ahead since we're done with len
// move 8 bytes ahead since we're done with len
let mut ptr = data . as_ptr ( ) . add ( 8 ) ; let mut ptr = data . as_ptr ( ) . add ( 8 ) ;
let end_ptr = data . as_ptr ( ) . add ( data . len ( ) ) ; let end_ptr = data . as_ptr ( ) . add ( data . len ( ) ) ;
for _ in 0 .. len { for _ in 0 .. len {
if ( ptr . add ( 8 ) ) > = end_ptr { if ( ptr . add ( 8 ) ) > = end_ptr {
// not enough space and even if there is a len
// not enough space and even if there is a len
// there is no value. This is even true for ZSTs
// there is no value. This is even true for ZSTs
return None ; return None ;
}
let lenkey = transmute_len ( ptr ) ;
ptr = ptr . add ( 8 ) ;
if ( ptr . add ( lenkey ) ) > end_ptr {
// not enough data left
return None ;
}
if ! T ::is_expected_len ( lenkey ) {
return None ;
}
// get the key as a raw slice, we've already checked if end_ptr is less
let key = T ::from_slice ( slice ::from_raw_parts ( ptr , lenkey ) ) ;
// move the ptr ahead; done with the key
ptr = ptr . add ( lenkey ) ;
// push it in
if ! set . insert ( key ) {
// repeat?; that's not what we wanted
return None ;
}
} }
let lenkey = transmute_len ( ptr ) ; if ptr = = end_ptr {
ptr = ptr . add ( 8 ) ; Some ( set )
if ( ptr . add ( lenkey ) ) > end_ptr { } else {
// not enough data left
// no pe, someone gave us more data
return None ; None
} }
if ! T ::is_expected_len ( lenkey ) { }
return None ; }
}
/// Deserializes a map-like set which has an 1B _bytemark_ for every entry
pub fn deserialize_set_ctype_bytemark < T > ( data : & [ u8 ] ) -> Option < HashMap < T , u8 > >
where
T : DeserializeFrom + Eq + Hash ,
{
// First read the length header
if data . len ( ) < 8 {
// so the file doesn't even have the length header? noice, just return
None
} else {
unsafe {
// so we have 8B. Just unsafe access and transmute it
let len = transmute_len ( data . as_ptr ( ) ) ;
let mut set = HashMap ::with_capacity ( len ) ;
// this is what we have left: [KLEN:8B]*
// move 8 bytes ahead since we're done with len
let mut ptr = data . as_ptr ( ) . add ( 8 ) ;
let end_ptr = data . as_ptr ( ) . add ( data . len ( ) ) ;
for _ in 0 .. len {
if ( ptr . add ( 8 ) ) > = end_ptr {
// not enough space and even if there is a len
// there is no value. This is even true for ZSTs
return None ;
}
let lenkey = transmute_len ( ptr ) ;
ptr = ptr . add ( 8 ) ;
if ( ptr . add ( lenkey + 1 ) ) > end_ptr {
// not enough data left
return None ;
}
if ! T ::is_expected_len ( lenkey ) {
return None ;
}
// get the key as a raw slice, we've already checked if end_ptr is less
let key = T ::from_slice ( slice ::from_raw_parts ( ptr , lenkey ) ) ;
// move the ptr ahead; done with the key
ptr = ptr . add ( lenkey ) ;
let bytemark = ptr ::read ( ptr ) ;
ptr = ptr . add ( 1 ) ;
// push it in
if set . insert ( key , bytemark ) . is_some ( ) {
// repeat?; that's not what we wanted
return None ;
}
} }
// get the key as a raw slice, we've already checked if end_ptr is less
if ptr = = end_ptr {
let key = T ::from_slice ( slice ::from_raw_parts ( ptr , lenkey ) ) ; Some ( set )
// move the ptr ahead; done with the key
} else {
ptr = ptr . add ( lenkey ) ; // nope, someone gave us more data
// push it in
None
if ! set . insert ( key ) {
// repeat?; that's not what we wanted
return None ;
} }
} }
if ptr = = end_ptr {
Some ( set )
} else {
// nope, someone gave us more data
None
}
} }
} }
} /// Deserialize a file that contains a serialized map. This also returns the model code
pub fn deserialize_map ( data : Vec < u8 > ) -> Option < ( Coremap < Data , Data > , u8 ) > {
/// Deserialize a file that contains a serialized map
// First read the length header
pub fn deserialize_map ( data : Vec < u8 > ) -> Option < Coremap < Data , Data > > { if data . len ( ) < 9 {
// First read the length header
// so the file doesn't even have the length/model header? noice, just return
if data . len ( ) < 8 { None
// so the file doesn't even have the length header? noice, just return
} else {
None unsafe {
} else { /*
unsafe { UNSAFE ( @ ohsayan ) : Everything done here is unsafely safe . We
/* reinterpret bits of one type as another . What could be worse ?
UNSAFE ( @ ohsayan ) : Everything done here is unsafely safe . We nah , it ' s not that bad . We know that the byte representations
reinterpret bits of one type as another . What could be worse ? would be in the way we expect . If the data is corrupted , we
nah , it ' s not that bad . We know that the byte representations can guarantee that we won ' t ever read incorrect lengths of data
would be in the way we expect . If the data is corrupted , we and we won ' t read into others ' memory ( or corrupt our own )
can guarantee that we won ' t ever read incorrect lengths of data * /
and we won ' t read into others ' memory ( or corrupt our own ) let mut ptr = data . as_ptr ( ) ;
* / let modelcode : u8 = ptr ::read ( ptr ) ;
// so we have 8B. Just unsafe access and transmute it; nobody cares
// model check
let len = transmute_len ( data . as_ptr ( ) ) ; if modelcode > 3 {
let hm = Coremap ::with_capacity ( len ) ; // this model isn't supposed to have more than 3. Corrupted data
// this is what we have left: [KLEN:8B][VLEN:8B]
// move 8 bytes ahead since we're done with len
let mut ptr = data . as_ptr ( ) . add ( 8 ) ;
let end_ptr = data . as_ptr ( ) . add ( data . len ( ) ) ;
for _ in 0 .. len {
if ( ptr . add ( 16 ) ) > = end_ptr {
// not enough space
return None ; return None ;
} }
let lenkey = transmute_len ( ptr ) ;
ptr = ptr . add ( 8 ) ; ptr = ptr . add ( 1 ) ;
let lenval = transmute_len ( ptr ) ; // so we have 8B. Just unsafe access and transmute it; nobody cares
let len = transmute_len ( ptr ) ;
// move 8 bytes ahead since we're done with len
ptr = ptr . add ( 8 ) ; ptr = ptr . add ( 8 ) ;
if ( ptr . add ( lenkey + lenval ) ) > end_ptr { let hm = Coremap ::with_capacity ( len ) ;
// not enough data left
// this is what we have left: [KLEN:8B][VLEN:8B]
return None ; let end_ptr = data . as_ptr ( ) . add ( data . len ( ) ) ;
for _ in 0 .. len {
if ( ptr . add ( 16 ) ) > = end_ptr {
// not enough space
return None ;
}
let lenkey = transmute_len ( ptr ) ;
ptr = ptr . add ( 8 ) ;
let lenval = transmute_len ( ptr ) ;
ptr = ptr . add ( 8 ) ;
if ( ptr . add ( lenkey + lenval ) ) > end_ptr {
// not enough data left
return None ;
}
// get the key as a raw slice, we've already checked if end_ptr is less
let key = Data ::copy_from_slice ( slice ::from_raw_parts ( ptr , lenkey ) ) ;
// move the ptr ahead; done with the key
ptr = ptr . add ( lenkey ) ;
let val = Data ::copy_from_slice ( slice ::from_raw_parts ( ptr , lenval ) ) ;
// move the ptr ahead; done with the value
ptr = ptr . add ( lenval ) ;
// push it in
hm . upsert ( key , val ) ;
}
if ptr = = end_ptr {
Some ( ( hm , modelcode ) )
} else {
// nope, someone gave us more data
None
} }
// get the key as a raw slice, we've already checked if end_ptr is less
let key = Data ::copy_from_slice ( slice ::from_raw_parts ( ptr , lenkey ) ) ;
// move the ptr ahead; done with the key
ptr = ptr . add ( lenkey ) ;
let val = Data ::copy_from_slice ( slice ::from_raw_parts ( ptr , lenval ) ) ;
// move the ptr ahead; done with the value
ptr = ptr . add ( lenval ) ;
// push it in
hm . upsert ( key , val ) ;
}
if ptr = = end_ptr {
Some ( hm )
} else {
// nope, someone gave us more data
None
} }
} }
} }
}
#[ allow(clippy::needless_return) ] // Clippy really misunderstands this
#[ allow(clippy::needless_return) ] // Clippy really misunderstands this
unsafe fn transmute_len ( start_ptr : * const u8 ) -> usize { unsafe fn transmute_len ( start_ptr : * const u8 ) -> usize {
little_endian ! ( { little_endian ! ( {
// So we have an LE target
// So we have an LE target
is_64_bit ! ( { is_64_bit ! ( {
// 64-bit LE
// 64-bit LE
return ptr ::read_unaligned ( start_ptr . cast ( ) ) ; return ptr ::read_unaligned ( start_ptr . cast ( ) ) ;
} ) ; } ) ;
not_64_bit ! ( { not_64_bit ! ( {
// 32-bit LE
// 32-bit LE
let ret1 : u64 = ptr ::read_unaligned ( start_ptr . cast ( ) ) ; let ret1 : u64 = ptr ::read_unaligned ( start_ptr . cast ( ) ) ;
// lossy cast
// lossy cast
let ret = ret1 as usize ; let ret = ret1 as usize ;
if ret > ( isize ::MAX as usize ) { if ret > ( isize ::MAX as usize ) {
// this is a backup method for us incase a giant 48-bit address is
// this is a backup method for us incase a giant 48-bit address is
// somehow forced to be read on this machine
// somehow forced to be read on this machine
panic! ( "RT panic: Very high size for current pointer width" ) ; panic! ( "RT panic: Very high size for current pointer width" ) ;
} }
return ret ; return ret ;
} ) ;
} ) ; } ) ;
} ) ;
big_endian ! ( { big_endian ! ( {
// so we have a BE target
// so we have a BE target
is_64_bit ! ( { is_64_bit ! ( {
// 64-bit big endian
// 64-bit big endian
let ret : usize = ptr ::read_unaligned ( start_ptr . cast ( ) ) ; let ret : usize = ptr ::read_unaligned ( start_ptr . cast ( ) ) ;
// swap byte order
// swap byte order
return ret . swap_bytes ( ) ; return ret . swap_bytes ( ) ;
} ) ; } ) ;
not_64_bit ! ( { not_64_bit ! ( {
// 32-bit big endian
// 32-bit big endian
let ret : u64 = ptr ::read_unaligned ( start_ptr . cast ( ) ) ; let ret : u64 = ptr ::read_unaligned ( start_ptr . cast ( ) ) ;
// swap byte order and lossy cast
// swap byte order and lossy cast
let ret = ( ret . swap_bytes ( ) ) as usize ; let ret = ( ret . swap_bytes ( ) ) as usize ;
// check if overflow
// check if overflow
if ret > ( isize ::MAX as usize ) { if ret > ( isize ::MAX as usize ) {
// this is a backup method for us incase a giant 48-bit address is
// this is a backup method for us incase a giant 48-bit address is
// somehow forced to be read on this machine
// somehow forced to be read on this machine
panic! ( "RT panic: Very high size for current pointer width" ) ; panic! ( "RT panic: Very high size for current pointer width" ) ;
} }
return ret ; return ret ;
} ) ;
} ) ; } ) ;
} ) ; }
} }
Sayan Nandan
3 years ago