gvsafronov
/
skytable
1
0
Fork 0
Code Pull Requests Activity

storage: Test database state post crash and recovery

Browse Source
next
Sayan Nandan 6 months ago
parent bb4132a617
commit af7e567b31
No known key found for this signature in database
GPG Key ID: 0EBD769024B24F0A
2 changed files with 404 additions and 113 deletions
Show Stats Download Patch File Download Diff File

4
server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs
Unescape Escape View File

@ -27,8 +27,8 @@
use {
super::{
super::{
create_journal, debug_get_trace, open_journal, DriverEventKind, JournalReaderTraceEvent,
JournalSettings, JournalWriterTraceEvent, RawJournalWriter,
create_journal, debug_get_trace, open_journal, DriverEventKind,
JournalReaderTraceEvent, JournalSettings, JournalWriterTraceEvent, RawJournalWriter,
},
SimpleDB, SimpleDBJournal,
},

513
server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs
Unescape Escape View File

@ -53,35 +53,82 @@ use {
};
const TRIALS: usize = 100;
const POST_TRIALS_SIZE: usize = TRIALS - (TRIALS / 10);
struct Initializer {
journal_id: &'static str,
initializer_fn: fn(&str) -> RuntimeResult<u64>,
initializer_fn: fn(&str) -> RuntimeResult<InitializerInfo>,
last_event_size: usize,
}
#[derive(Debug)]
struct ModifiedJournalInfo {
init: InitializerInfo,
_storage: ModifiedJournalStorageInfo,
initializer_id: usize,
}
impl ModifiedJournalInfo {
fn new(
init: InitializerInfo,
storage: ModifiedJournalStorageInfo,
initializer_id: usize,
) -> Self {
Self {
init,
_storage: storage,
initializer_id,
}
}
}
#[derive(Debug, Clone, Copy)]
struct InitializerInfo {
corrupted_event_id: u64,
last_executed_event_id: u64,
}
impl InitializerInfo {
fn new_last_event(last_event_id: u64) -> Self {
Self::new(last_event_id, last_event_id)
}
fn new(corrupted_event_id: u64, last_executed_event_id: u64) -> Self {
Self {
corrupted_event_id,
last_executed_event_id,
}
}
fn not_last_event(&self) -> bool {
self.corrupted_event_id != self.last_executed_event_id
}
}
impl Initializer {
fn new(name: &'static str, f: fn(&str) -> RuntimeResult<u64>, last_event_size: usize) -> Self {
fn new(
name: &'static str,
f: fn(&str) -> RuntimeResult<InitializerInfo>,
last_event_size: usize,
) -> Self {
Self {
journal_id: name,
initializer_fn: f,
last_event_size,
}
}
fn new_driver_type(name: &'static str, f: fn(&str) -> RuntimeResult<u64>) -> Self {
fn new_driver_type(name: &'static str, f: fn(&str) -> RuntimeResult<InitializerInfo>) -> Self {
Self::new(name, f, DriverEvent::FULL_EVENT_SIZE)
}
}
#[derive(Debug)]
#[allow(unused)]
struct ModifiedJournalInfo {
struct ModifiedJournalStorageInfo {
original_file_size: usize,
modified_file_size: usize,
corruption_range: Range<usize>,
}
impl ModifiedJournalInfo {
impl ModifiedJournalStorageInfo {
fn new(
original_file_size: usize,
modified_file_size: usize,
@ -100,35 +147,37 @@ fn emulate_sequentially_varying_single_corruption(
modified_journal_generator_fn: impl Fn(
&str,
&str,
u64,
&InitializerInfo,
usize,
&BTreeMap<u64, u64>,
) -> IoResult<ModifiedJournalInfo>,
) -> IoResult<ModifiedJournalStorageInfo>,
post_corruption_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
SimpleDB,
RuntimeResult<RawJournalWriter<SimpleDBJournal>>,
ModifiedJournalInfo,
),
post_repair_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
RuntimeResult<RepairResult>,
SimpleDB,
RuntimeResult<RawJournalWriter<SimpleDBJournal>>,
),
) {
for Initializer {
journal_id,
initializer_fn,
last_event_size,
} in initializers
for (
initializer_id,
Initializer {
journal_id,
initializer_fn,
last_event_size,
},
) in initializers.into_iter().enumerate()
{
// initialize journal, get size and clear traces
let repaired_last_event_id = match initializer_fn(journal_id) {
let initializer_info = match initializer_fn(journal_id) {
Ok(nid) => nid,
Err(e) => panic!(
"failed to initialize {journal_id} due to {e}. trace: {:?}, file_data={:?}",
@ -150,41 +199,46 @@ fn emulate_sequentially_varying_single_corruption(
)
};
// modify journal
let mod_stat = modified_journal_generator_fn(
let storage_info = modified_journal_generator_fn(
journal_id,
&corrupted_journal_path,
repaired_last_event_id,
&initializer_info,
trim_size,
&original_offsets,
)
.unwrap();
let modified_journal_info =
ModifiedJournalInfo::new(initializer_info, storage_info, initializer_id);
// now let the caller handle any post corruption work
{
let sdb = SimpleDB::new();
let open_journal_result = open_journal_fn(&sdb);
post_corruption_handler(
journal_id,
repaired_last_event_id,
&modified_journal_info,
trim_size,
sdb,
open_journal_result,
mod_stat,
);
}
// repair and let the caller handle post repair work
let repair_result;
{
let sdb = SimpleDB::new();
let repair_result = repair_journal::<SimpleDBJournal>(
repair_result = repair_journal::<SimpleDBJournal>(
&corrupted_journal_path,
&sdb,
JournalSettings::default(),
JournalRepairMode::Simple,
);
}
{
let sdb = SimpleDB::new();
let repaired_journal_reopen_result = open_journal_fn(&sdb);
// let caller handle any post repair work
post_repair_handler(
journal_id,
repaired_last_event_id,
&modified_journal_info,
trim_size,
repair_result,
sdb,
@ -199,15 +253,14 @@ fn emulate_final_event_corruption(
initializers: impl IntoIterator<Item = Initializer>,
post_corruption_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
SimpleDB,
RuntimeResult<RawJournalWriter<SimpleDBJournal>>,
ModifiedJournalInfo,
),
post_repair_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
RuntimeResult<RepairResult>,
SimpleDB,
@ -221,7 +274,7 @@ fn emulate_final_event_corruption(
let mut f = File::open(modified_journal)?;
let real_flen = f.f_len()? as usize;
f.f_truncate((real_flen - trim_amount) as _)?;
Ok(ModifiedJournalInfo::new(
Ok(ModifiedJournalStorageInfo::new(
real_flen,
trim_amount,
trim_amount..real_flen,
@ -236,15 +289,14 @@ fn emulate_midway_corruption(
initializers: impl IntoIterator<Item = Initializer>,
post_corruption_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
SimpleDB,
RuntimeResult<RawJournalWriter<SimpleDBJournal>>,
ModifiedJournalInfo,
),
post_repair_handler: impl Fn(
&str,
u64,
&ModifiedJournalInfo,
usize,
RuntimeResult<RepairResult>,
SimpleDB,
@ -255,13 +307,15 @@ fn emulate_midway_corruption(
initializers,
|original_journal_path,
corrupted_journal_path,
event_to_corrupt,
initializer_info,
trim_size,
original_offsets| {
let orig_journal_data = FileSystem::read(original_journal_path)?;
let orig_journal_size = orig_journal_data.len();
let mut f = File::create(corrupted_journal_path)?;
let end_offset = *original_offsets.get(&event_to_corrupt).unwrap() as usize;
let end_offset = *original_offsets
.get(&initializer_info.corrupted_event_id)
.unwrap() as usize;
// apply
let segment_before_corruption = &orig_journal_data[..end_offset - trim_size];
let segment_after_corruption = &orig_journal_data[end_offset..];
@ -276,7 +330,7 @@ fn emulate_midway_corruption(
);
f.fwrite_all(segment_before_corruption)?;
f.fwrite_all(segment_after_corruption)?;
Ok(ModifiedJournalInfo::new(
Ok(ModifiedJournalStorageInfo::new(
orig_journal_size,
new_size,
end_offset - trim_size..end_offset,
@ -287,17 +341,22 @@ fn emulate_midway_corruption(
)
}
fn keyfmt(num: usize) -> String {
format!("key-{num:06}")
}
fn apply_event_mix(jrnl: &mut RawJournalWriter<SimpleDBJournal>) -> RuntimeResult<u64> {
let mut op_count = 0;
let mut sdb = SimpleDB::new();
for num in 1..=TRIALS {
op_count += 1;
sdb.push(jrnl, format!("key-{num:06}"))?;
sdb.push(jrnl, keyfmt(num))?;
if num % 10 == 0 {
op_count += 1;
sdb.pop(jrnl)?;
}
}
assert_eq!(sdb.data().len(), POST_TRIALS_SIZE);
Ok(op_count)
}
@ -308,14 +367,14 @@ fn corruption_before_close() {
Initializer::new_driver_type("close_event_corruption_empty.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
Ok(0)
Ok(InitializerInfo::new_last_event(0))
}),
// open, apply mix of events, close
Initializer::new_driver_type("close_event_corruption.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
let operation_count = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
Ok(operation_count)
Ok(InitializerInfo::new_last_event(operation_count))
}),
// open, close, reinit, close
Initializer::new_driver_type(
@ -332,20 +391,25 @@ fn corruption_before_close() {
JournalSettings::default(),
)?;
RawJournalWriter::close_driver(&mut jrnl)?;
Ok(2)
Ok(InitializerInfo::new_last_event(2))
},
),
];
emulate_final_event_corruption(
initializers,
|journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| {
|journal_id, modified_journal_info, trim_size, db, open_result| {
// open the journal and validate failure
let open_err = open_result.unwrap_err();
let trace = debug_get_trace();
if trim_size > (DriverEvent::FULL_EVENT_SIZE - (sizeof!(u128) + sizeof!(u64))) {
// the amount of trim from the end of the file causes us to lose valuable metadata
if repaired_last_event_id == 0 {
if modified_journal_info.init.last_executed_event_id == 0 {
// empty log
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
trace,
intovec![
@ -355,6 +419,25 @@ fn corruption_before_close() {
"failed at trim_size {trim_size} for journal {journal_id}"
)
} else {
if modified_journal_info.initializer_id == 1 {
// in the second case, we apply the event mix so we need to check this
assert_eq!(
db.data().len(),
POST_TRIALS_SIZE,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
} else {
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
}
assert_eq!(
*trace.last().unwrap(),
JournalReaderTraceEvent::LookingForEvent.into(),
@ -363,24 +446,52 @@ fn corruption_before_close() {
}
} else {
// the amount of trim still allows us to read some metadata
if repaired_last_event_id == 0 {
if modified_journal_info.init.last_executed_event_id == 0 {
// empty log
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id),
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id
),
JournalReaderTraceEvent::DriverEventExpectingClose,
],
"failed at trim_size {trim_size} for journal {journal_id}"
)
} else {
if modified_journal_info.initializer_id == 1 {
// in the second case, we apply the event mix so we need to check this
assert_eq!(
db.data().len(),
POST_TRIALS_SIZE,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
} else {
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
}
assert_eq!(
&trace[trace.len() - 3..],
&into_array![
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id),
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id
),
JournalReaderTraceEvent::DriverEventExpectingClose
],
"failed at trim_size {trim_size} for journal {journal_id}"
@ -393,12 +504,33 @@ fn corruption_before_close() {
"failed at trim_size {trim_size} for journal {journal_id}"
);
},
|journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| {
|journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| {
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss((DriverEvent::FULL_EVENT_SIZE - trim_size) as _),
"failed at trim_size {trim_size} for journal {journal_id}"
);
if modified_journal_info.init.last_executed_event_id == 0
|| modified_journal_info.initializer_id == 2
{
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
} else {
// in the second case, we apply the event mix so we need to check this
assert_eq!(
db.data().len(),
POST_TRIALS_SIZE,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
}
let _ = reopen_result.unwrap();
// clear trace
let _ = debug_get_trace();
@ -416,7 +548,7 @@ fn corruption_after_reopen() {
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(1)
Ok(InitializerInfo::new_last_event(1))
}),
Initializer::new_driver_type("corruption_after_ropen_multi_before_close.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
@ -425,12 +557,12 @@ fn corruption_after_reopen() {
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(operation_count + 1) // + 1 since we have the reopen event which is the next event that'll vanish
Ok(InitializerInfo::new_last_event(operation_count + 1)) // + 1 since we have the reopen event which is the next event that'll vanish
}),
];
emulate_final_event_corruption(
initializers,
|journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| {
|journal_id, modified_journal_info, trim_size, db, open_result| {
let trace = debug_get_trace();
if trim_size == DriverEvent::FULL_EVENT_SIZE {
/*
@ -444,8 +576,13 @@ fn corruption_after_reopen() {
*/
let mut jrnl =
open_result.expect(&format!("failed at {trim_size} for journal {journal_id}"));
if repaired_last_event_id == 1 {
if modified_journal_info.init.last_executed_event_id == 1 {
// empty log, only the reopen
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
trace,
intovec![
@ -460,7 +597,7 @@ fn corruption_after_reopen() {
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
event_id: modified_journal_info.init.corrupted_event_id,
prev_id: 0
},
JournalWriterTraceEvent::DriverEventCompleted,
@ -469,12 +606,25 @@ fn corruption_after_reopen() {
"failed at {trim_size} for journal {journal_id}"
);
} else {
// we will have upto the last event since only the reopen is gone
assert_eq!(
db.data().len(),
POST_TRIALS_SIZE,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
&trace[trace.len() - 12..],
intovec![
JournalReaderTraceEvent::ServerEventAppliedSuccess,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1), // close event
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id - 1
), // close event
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
@ -483,12 +633,13 @@ fn corruption_after_reopen() {
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: repaired_last_event_id - 1 // close event
event_id: modified_journal_info.init.corrupted_event_id,
prev_id: modified_journal_info.init.corrupted_event_id - 1 // close event
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete
]
],
"failed at {trim_size} for journal {journal_id}"
)
}
// now close this so that this works with the post repair handler
@ -498,10 +649,16 @@ fn corruption_after_reopen() {
} else {
assert_eq!(
open_result.unwrap_err().kind(),
&ErrorKind::IoError(IoErrorKind::UnexpectedEof.into())
&ErrorKind::IoError(IoErrorKind::UnexpectedEof.into()),
"failed at {trim_size} for journal {journal_id}"
);
if repaired_last_event_id == 1 {
if modified_journal_info.init.last_executed_event_id == 1 {
// empty log, only the reopen
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
assert_eq!(
trace,
intovec![
@ -512,10 +669,16 @@ fn corruption_after_reopen() {
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id
)
],
"failed at {trim_size} for journal {journal_id}"
);
} else {
// we will have upto the last event since only the reopen is gone
assert_eq!(db.data().len(), POST_TRIALS_SIZE);
assert_eq!(*db.data().last().unwrap(), keyfmt(TRIALS - 1));
assert_eq!(
&trace[trace.len() - 5..],
intovec![
@ -523,13 +686,16 @@ fn corruption_after_reopen() {
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id
)
],
"failed at {trim_size} for journal {journal_id}"
);
}
}
},
|journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| {
|journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| {
assert!(reopen_result.is_ok());
if trim_size == DriverEvent::FULL_EVENT_SIZE {
// see earlier comment
@ -543,9 +709,20 @@ fn corruption_after_reopen() {
repair_result.unwrap(),
RepairResult::UnspecifiedLoss(
(DriverEvent::FULL_EVENT_SIZE - trim_size) as u64
)
),
"failed at {trim_size} for journal {journal_id}"
);
}
if modified_journal_info.init.last_executed_event_id == 1 {
assert_eq!(
db.data().len(),
0,
"failed at {trim_size} for journal {journal_id}"
);
} else {
assert_eq!(db.data().len(), POST_TRIALS_SIZE);
assert_eq!(*db.data().last().unwrap(), keyfmt(TRIALS - 1));
}
let _ = debug_get_trace();
let _ = debug_get_offsets();
},
@ -576,7 +753,7 @@ fn corruption_at_runtime() {
let mut jrnl = create_journal(jrnl_id)?;
sdb.push(&mut jrnl, KEY)?;
// don't close
Ok(0)
Ok(InitializerInfo::new_last_event(0))
},
offset,
),
@ -586,19 +763,19 @@ fn corruption_at_runtime() {
let mut op_count = 0;
let mut sdb = SimpleDB::new();
let mut jrnl = create_journal(jrnl_id)?;
for _ in 0..TRIALS {
for _ in 1..=TRIALS {
sdb.push(&mut jrnl, KEY)?;
op_count += 1;
}
// don't close
Ok(op_count)
Ok(InitializerInfo::new_last_event(op_count))
},
offset,
),
];
emulate_final_event_corruption(
initializers,
|journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| {
|journal_id, modified_journal_info, trim_size, db, open_result| {
let trace = debug_get_trace();
let err = open_result.unwrap_err();
assert_eq!(
@ -607,7 +784,12 @@ fn corruption_at_runtime() {
"failed for journal {journal_id} with trim_size {trim_size}"
);
if trim_size > offset - (sizeof!(u128) + sizeof!(u64)) {
if repaired_last_event_id == 0 {
if modified_journal_info.init.last_executed_event_id == 0 {
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
trace,
intovec![
@ -617,6 +799,17 @@ fn corruption_at_runtime() {
"failed for journal {journal_id} with trim_size {trim_size}"
)
} else {
// we lost the last server event, so we'll have one key less
assert_eq!(
db.data().len(),
TRIALS - 1,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
db.data()[TRIALS - 2],
KEY,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
&trace[trace.len() - 4..],
intovec![
@ -629,8 +822,13 @@ fn corruption_at_runtime() {
)
}
} else {
if repaired_last_event_id == 0 {
if modified_journal_info.init.last_executed_event_id == 0 {
// empty log
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
trace,
intovec![
@ -643,11 +841,24 @@ fn corruption_at_runtime() {
"failed for journal {journal_id} with trim_size {trim_size}"
);
} else {
// we lost the last server event, so we'll have one key less
assert_eq!(
db.data().len(),
TRIALS - 1,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
db.data()[TRIALS - 2],
KEY,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
&trace[trace.len() - 4..],
intovec![
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1),
JournalReaderTraceEvent::AttemptingEvent(
modified_journal_info.init.corrupted_event_id - 1
),
JournalReaderTraceEvent::DetectedServerEvent,
JournalReaderTraceEvent::ServerEventMetadataParsed,
],
@ -656,13 +867,31 @@ fn corruption_at_runtime() {
}
}
},
|journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| {
|journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| {
assert!(reopen_result.is_ok());
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss((offset - trim_size) as u64),
"failed for journal {journal_id} with trim_size {trim_size}"
);
if modified_journal_info.init.last_executed_event_id == 0 {
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
} else {
assert_eq!(
db.data().len(),
TRIALS - 1,
"failed for journal {journal_id} with trim_size {trim_size}"
);
assert_eq!(
db.data()[TRIALS - 2],
KEY,
"failed for journal {journal_id} with trim_size {trim_size}"
);
}
let _ = debug_get_trace();
},
)
@ -684,7 +913,7 @@ fn midway_corruption_close() {
)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
Ok(0) // close (to corrupt), reopen, close
Ok(InitializerInfo::new(0, 2)) // close (to corrupt), reopen, close
}),
Initializer::new_driver_type(
"midway_corruption_close_events_before_second_close",
@ -692,7 +921,18 @@ fn midway_corruption_close() {
{
// create and close
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?; // (0)
}
let op_cnt;
{
// reopen, apply mix and close
let mut jrnl = open_journal::<SimpleDBJournal>(
jrnl_id,
&SimpleDB::new(),
JournalSettings::default(),
)?; // (1)
op_cnt = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?; // <-- (op_cnt + 2) corrupt this one
}
{
// reopen and close
@ -700,69 +940,120 @@ fn midway_corruption_close() {
jrnl_id,
&SimpleDB::new(),
JournalSettings::default(),
)?;
RawJournalWriter::close_driver(&mut jrnl)?; // <-- corrupt this one
)?; // (op_cnt + 3)
RawJournalWriter::close_driver(&mut jrnl)?; // (op_cnt + 4)
}
Ok(InitializerInfo::new(op_cnt + 2, op_cnt + 4))
},
),
Initializer::new_driver_type(
"midway_corruption_close_events_before_third_close",
|jrnl_id| {
{
// create and close
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?; // (0)
}
{
// reopen and close
let mut jrnl = open_journal::<SimpleDBJournal>(
jrnl_id,
&SimpleDB::new(),
JournalSettings::default(),
)?;
let _ = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
)?; // (1)
RawJournalWriter::close_driver(&mut jrnl)?; // <-- (2) corrupt this one
}
Ok(2) // corrupt the second close event
let op_cnt;
{
let mut jrnl = open_journal::<SimpleDBJournal>(
jrnl_id,
&SimpleDB::new(),
JournalSettings::default(),
)?; // (3)
op_cnt = apply_event_mix(&mut jrnl)?; // (3 + op_count)
RawJournalWriter::close_driver(&mut jrnl)?; // (4 + op_count)
}
Ok(InitializerInfo::new(2, op_cnt + 4)) // corrupt the second close event
},
),
];
debug_set_offset_tracking(true);
emulate_midway_corruption(
initializers,
|journal_id, _last_id, trim_size, db, open_result, _modstat| {
|journal_id, modified_journal_info, trim_size, db, open_result| {
assert!(
open_result.is_err(),
"failed for journal {journal_id} with trim_size {trim_size}"
);
// all data will be lost, so the DB will be empty
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
match modified_journal_info.initializer_id {
0 | 2 => {
// in the first and third case, (0) no data is present (2) all data is lost
// all data will be lost, so the DB will be empty
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
}
1 => {
// in this case, all elements will be preserved
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
}
_ => panic!(),
}
let _ = debug_get_offsets();
let _ = debug_get_trace();
},
|journal_id, last_id, trim_size, repair_result, db, reopen_result| {
|journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| {
let _ = reopen_result.unwrap();
// all data will be lost, so the DB will be empty
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
if last_id == 0 {
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss(
((DriverEvent::FULL_EVENT_SIZE * 3) - trim_size) as u64
),
"failed for journal {journal_id} with trim_size {trim_size}"
);
} else {
// this is a serious midway corruption with major data loss
let full_log_size = File::open(journal_id).unwrap().f_len().unwrap();
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss(
full_log_size
- <<SimpleDBJournal as RawJournalAdapter>::Spec as FileSpecV1>::SIZE // account for header
as u64
- (DriverEvent::FULL_EVENT_SIZE * 2) as u64 // account for close (0), reopen(1)
- trim_size as u64 // account for trim
),
"failed for journal {journal_id} with trim_size {trim_size}"
);
match modified_journal_info.initializer_id {
0 | 2 => {
// all data will be lost, so the DB will be empty
assert_eq!(
db.data().len(),
0,
"failed for journal {journal_id} with trim_size {trim_size}"
);
if modified_journal_info.init.corrupted_event_id == 0
&& modified_journal_info.init.not_last_event()
{
// the first event was corrupted
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss(
((DriverEvent::FULL_EVENT_SIZE * 3) - trim_size) as u64
),
"failed for journal {journal_id} with trim_size {trim_size}"
);
} else {
// this is a serious midway corruption with major data loss
let full_log_size = File::open(journal_id).unwrap().f_len().unwrap();
assert_eq!(
repair_result.unwrap(),
RepairResult::UnspecifiedLoss(
full_log_size
- <<SimpleDBJournal as RawJournalAdapter>::Spec as FileSpecV1>::SIZE // account for header
as u64
- (DriverEvent::FULL_EVENT_SIZE * 2) as u64 // account for close (0), reopen(1)
- trim_size as u64 // account for trim
),
"failed for journal {journal_id} with trim_size {trim_size}"
);
}
}
1 => {
// in this case, all elements will be preserved
assert_eq!(
*db.data().last().unwrap(),
keyfmt(TRIALS - 1),
"failed at {trim_size} for journal {journal_id}"
);
}
_ => panic!(),
}
let _ = debug_get_trace();
let _ = debug_get_offsets();

Write Preview
Loading…
Cancel
Save