diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs b/server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs index 7729d403..bb892d22 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs @@ -27,8 +27,8 @@ use { super::{ super::{ - create_journal, debug_get_trace, open_journal, DriverEventKind, JournalReaderTraceEvent, - JournalSettings, JournalWriterTraceEvent, RawJournalWriter, + create_journal, debug_get_trace, open_journal, DriverEventKind, + JournalReaderTraceEvent, JournalSettings, JournalWriterTraceEvent, RawJournalWriter, }, SimpleDB, SimpleDBJournal, }, diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs b/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs index ace94d8d..0af5188e 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs @@ -53,35 +53,82 @@ use { }; const TRIALS: usize = 100; +const POST_TRIALS_SIZE: usize = TRIALS - (TRIALS / 10); struct Initializer { journal_id: &'static str, - initializer_fn: fn(&str) -> RuntimeResult, + initializer_fn: fn(&str) -> RuntimeResult, last_event_size: usize, } +#[derive(Debug)] +struct ModifiedJournalInfo { + init: InitializerInfo, + _storage: ModifiedJournalStorageInfo, + initializer_id: usize, +} + +impl ModifiedJournalInfo { + fn new( + init: InitializerInfo, + storage: ModifiedJournalStorageInfo, + initializer_id: usize, + ) -> Self { + Self { + init, + _storage: storage, + initializer_id, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct InitializerInfo { + corrupted_event_id: u64, + last_executed_event_id: u64, +} + +impl InitializerInfo { + fn new_last_event(last_event_id: u64) -> Self { + Self::new(last_event_id, last_event_id) + } + fn new(corrupted_event_id: u64, last_executed_event_id: u64) -> Self { + Self { + corrupted_event_id, + last_executed_event_id, + } + } + fn not_last_event(&self) -> bool { + self.corrupted_event_id != self.last_executed_event_id + } +} + impl Initializer { - fn new(name: &'static str, f: fn(&str) -> RuntimeResult, last_event_size: usize) -> Self { + fn new( + name: &'static str, + f: fn(&str) -> RuntimeResult, + last_event_size: usize, + ) -> Self { Self { journal_id: name, initializer_fn: f, last_event_size, } } - fn new_driver_type(name: &'static str, f: fn(&str) -> RuntimeResult) -> Self { + fn new_driver_type(name: &'static str, f: fn(&str) -> RuntimeResult) -> Self { Self::new(name, f, DriverEvent::FULL_EVENT_SIZE) } } #[derive(Debug)] #[allow(unused)] -struct ModifiedJournalInfo { +struct ModifiedJournalStorageInfo { original_file_size: usize, modified_file_size: usize, corruption_range: Range, } -impl ModifiedJournalInfo { +impl ModifiedJournalStorageInfo { fn new( original_file_size: usize, modified_file_size: usize, @@ -100,35 +147,37 @@ fn emulate_sequentially_varying_single_corruption( modified_journal_generator_fn: impl Fn( &str, &str, - u64, + &InitializerInfo, usize, &BTreeMap, - ) -> IoResult, + ) -> IoResult, post_corruption_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, SimpleDB, RuntimeResult>, - ModifiedJournalInfo, ), post_repair_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, RuntimeResult, SimpleDB, RuntimeResult>, ), ) { - for Initializer { - journal_id, - initializer_fn, - last_event_size, - } in initializers + for ( + initializer_id, + Initializer { + journal_id, + initializer_fn, + last_event_size, + }, + ) in initializers.into_iter().enumerate() { // initialize journal, get size and clear traces - let repaired_last_event_id = match initializer_fn(journal_id) { + let initializer_info = match initializer_fn(journal_id) { Ok(nid) => nid, Err(e) => panic!( "failed to initialize {journal_id} due to {e}. trace: {:?}, file_data={:?}", @@ -150,41 +199,46 @@ fn emulate_sequentially_varying_single_corruption( ) }; // modify journal - let mod_stat = modified_journal_generator_fn( + let storage_info = modified_journal_generator_fn( journal_id, &corrupted_journal_path, - repaired_last_event_id, + &initializer_info, trim_size, &original_offsets, ) .unwrap(); + let modified_journal_info = + ModifiedJournalInfo::new(initializer_info, storage_info, initializer_id); // now let the caller handle any post corruption work { let sdb = SimpleDB::new(); let open_journal_result = open_journal_fn(&sdb); post_corruption_handler( journal_id, - repaired_last_event_id, + &modified_journal_info, trim_size, sdb, open_journal_result, - mod_stat, ); } // repair and let the caller handle post repair work + let repair_result; { let sdb = SimpleDB::new(); - let repair_result = repair_journal::( + repair_result = repair_journal::( &corrupted_journal_path, &sdb, JournalSettings::default(), JournalRepairMode::Simple, ); + } + { + let sdb = SimpleDB::new(); let repaired_journal_reopen_result = open_journal_fn(&sdb); // let caller handle any post repair work post_repair_handler( journal_id, - repaired_last_event_id, + &modified_journal_info, trim_size, repair_result, sdb, @@ -199,15 +253,14 @@ fn emulate_final_event_corruption( initializers: impl IntoIterator, post_corruption_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, SimpleDB, RuntimeResult>, - ModifiedJournalInfo, ), post_repair_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, RuntimeResult, SimpleDB, @@ -221,7 +274,7 @@ fn emulate_final_event_corruption( let mut f = File::open(modified_journal)?; let real_flen = f.f_len()? as usize; f.f_truncate((real_flen - trim_amount) as _)?; - Ok(ModifiedJournalInfo::new( + Ok(ModifiedJournalStorageInfo::new( real_flen, trim_amount, trim_amount..real_flen, @@ -236,15 +289,14 @@ fn emulate_midway_corruption( initializers: impl IntoIterator, post_corruption_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, SimpleDB, RuntimeResult>, - ModifiedJournalInfo, ), post_repair_handler: impl Fn( &str, - u64, + &ModifiedJournalInfo, usize, RuntimeResult, SimpleDB, @@ -255,13 +307,15 @@ fn emulate_midway_corruption( initializers, |original_journal_path, corrupted_journal_path, - event_to_corrupt, + initializer_info, trim_size, original_offsets| { let orig_journal_data = FileSystem::read(original_journal_path)?; let orig_journal_size = orig_journal_data.len(); let mut f = File::create(corrupted_journal_path)?; - let end_offset = *original_offsets.get(&event_to_corrupt).unwrap() as usize; + let end_offset = *original_offsets + .get(&initializer_info.corrupted_event_id) + .unwrap() as usize; // apply let segment_before_corruption = &orig_journal_data[..end_offset - trim_size]; let segment_after_corruption = &orig_journal_data[end_offset..]; @@ -276,7 +330,7 @@ fn emulate_midway_corruption( ); f.fwrite_all(segment_before_corruption)?; f.fwrite_all(segment_after_corruption)?; - Ok(ModifiedJournalInfo::new( + Ok(ModifiedJournalStorageInfo::new( orig_journal_size, new_size, end_offset - trim_size..end_offset, @@ -287,17 +341,22 @@ fn emulate_midway_corruption( ) } +fn keyfmt(num: usize) -> String { + format!("key-{num:06}") +} + fn apply_event_mix(jrnl: &mut RawJournalWriter) -> RuntimeResult { let mut op_count = 0; let mut sdb = SimpleDB::new(); for num in 1..=TRIALS { op_count += 1; - sdb.push(jrnl, format!("key-{num:06}"))?; + sdb.push(jrnl, keyfmt(num))?; if num % 10 == 0 { op_count += 1; sdb.pop(jrnl)?; } } + assert_eq!(sdb.data().len(), POST_TRIALS_SIZE); Ok(op_count) } @@ -308,14 +367,14 @@ fn corruption_before_close() { Initializer::new_driver_type("close_event_corruption_empty.db", |jrnl_id| { let mut jrnl = create_journal::(jrnl_id)?; RawJournalWriter::close_driver(&mut jrnl)?; - Ok(0) + Ok(InitializerInfo::new_last_event(0)) }), // open, apply mix of events, close Initializer::new_driver_type("close_event_corruption.db", |jrnl_id| { let mut jrnl = create_journal::(jrnl_id)?; let operation_count = apply_event_mix(&mut jrnl)?; RawJournalWriter::close_driver(&mut jrnl)?; - Ok(operation_count) + Ok(InitializerInfo::new_last_event(operation_count)) }), // open, close, reinit, close Initializer::new_driver_type( @@ -332,20 +391,25 @@ fn corruption_before_close() { JournalSettings::default(), )?; RawJournalWriter::close_driver(&mut jrnl)?; - Ok(2) + Ok(InitializerInfo::new_last_event(2)) }, ), ]; emulate_final_event_corruption( initializers, - |journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| { + |journal_id, modified_journal_info, trim_size, db, open_result| { // open the journal and validate failure let open_err = open_result.unwrap_err(); let trace = debug_get_trace(); if trim_size > (DriverEvent::FULL_EVENT_SIZE - (sizeof!(u128) + sizeof!(u64))) { // the amount of trim from the end of the file causes us to lose valuable metadata - if repaired_last_event_id == 0 { + if modified_journal_info.init.last_executed_event_id == 0 { // empty log + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); assert_eq!( trace, intovec![ @@ -355,6 +419,25 @@ fn corruption_before_close() { "failed at trim_size {trim_size} for journal {journal_id}" ) } else { + if modified_journal_info.initializer_id == 1 { + // in the second case, we apply the event mix so we need to check this + assert_eq!( + db.data().len(), + POST_TRIALS_SIZE, + "failed at {trim_size} for journal {journal_id}" + ); + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); + } else { + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); + } assert_eq!( *trace.last().unwrap(), JournalReaderTraceEvent::LookingForEvent.into(), @@ -363,24 +446,52 @@ fn corruption_before_close() { } } else { // the amount of trim still allows us to read some metadata - if repaired_last_event_id == 0 { + if modified_journal_info.init.last_executed_event_id == 0 { // empty log + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); assert_eq!( trace, intovec![ JournalReaderTraceEvent::Initialized, JournalReaderTraceEvent::LookingForEvent, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id), + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id + ), JournalReaderTraceEvent::DriverEventExpectingClose, ], "failed at trim_size {trim_size} for journal {journal_id}" ) } else { + if modified_journal_info.initializer_id == 1 { + // in the second case, we apply the event mix so we need to check this + assert_eq!( + db.data().len(), + POST_TRIALS_SIZE, + "failed at {trim_size} for journal {journal_id}" + ); + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); + } else { + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); + } assert_eq!( &trace[trace.len() - 3..], &into_array![ JournalReaderTraceEvent::LookingForEvent, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id), + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id + ), JournalReaderTraceEvent::DriverEventExpectingClose ], "failed at trim_size {trim_size} for journal {journal_id}" @@ -393,12 +504,33 @@ fn corruption_before_close() { "failed at trim_size {trim_size} for journal {journal_id}" ); }, - |journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| { + |journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| { assert_eq!( repair_result.unwrap(), RepairResult::UnspecifiedLoss((DriverEvent::FULL_EVENT_SIZE - trim_size) as _), "failed at trim_size {trim_size} for journal {journal_id}" ); + if modified_journal_info.init.last_executed_event_id == 0 + || modified_journal_info.initializer_id == 2 + { + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); + } else { + // in the second case, we apply the event mix so we need to check this + assert_eq!( + db.data().len(), + POST_TRIALS_SIZE, + "failed at {trim_size} for journal {journal_id}" + ); + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); + } let _ = reopen_result.unwrap(); // clear trace let _ = debug_get_trace(); @@ -416,7 +548,7 @@ fn corruption_after_reopen() { drop(jrnl); // reopen, but don't close open_journal::(jrnl_id, &SimpleDB::new(), JournalSettings::default())?; - Ok(1) + Ok(InitializerInfo::new_last_event(1)) }), Initializer::new_driver_type("corruption_after_ropen_multi_before_close.db", |jrnl_id| { let mut jrnl = create_journal::(jrnl_id)?; @@ -425,12 +557,12 @@ fn corruption_after_reopen() { drop(jrnl); // reopen, but don't close open_journal::(jrnl_id, &SimpleDB::new(), JournalSettings::default())?; - Ok(operation_count + 1) // + 1 since we have the reopen event which is the next event that'll vanish + Ok(InitializerInfo::new_last_event(operation_count + 1)) // + 1 since we have the reopen event which is the next event that'll vanish }), ]; emulate_final_event_corruption( initializers, - |journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| { + |journal_id, modified_journal_info, trim_size, db, open_result| { let trace = debug_get_trace(); if trim_size == DriverEvent::FULL_EVENT_SIZE { /* @@ -444,8 +576,13 @@ fn corruption_after_reopen() { */ let mut jrnl = open_result.expect(&format!("failed at {trim_size} for journal {journal_id}")); - if repaired_last_event_id == 1 { + if modified_journal_info.init.last_executed_event_id == 1 { // empty log, only the reopen + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); assert_eq!( trace, intovec![ @@ -460,7 +597,7 @@ fn corruption_after_reopen() { JournalWriterTraceEvent::ReinitializeAttempt, JournalWriterTraceEvent::DriverEventAttemptCommit { event: DriverEventKind::Reopened, - event_id: repaired_last_event_id, + event_id: modified_journal_info.init.corrupted_event_id, prev_id: 0 }, JournalWriterTraceEvent::DriverEventCompleted, @@ -469,12 +606,25 @@ fn corruption_after_reopen() { "failed at {trim_size} for journal {journal_id}" ); } else { + // we will have upto the last event since only the reopen is gone + assert_eq!( + db.data().len(), + POST_TRIALS_SIZE, + "failed at {trim_size} for journal {journal_id}" + ); + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); assert_eq!( &trace[trace.len() - 12..], intovec![ JournalReaderTraceEvent::ServerEventAppliedSuccess, JournalReaderTraceEvent::LookingForEvent, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1), // close event + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id - 1 + ), // close event JournalReaderTraceEvent::DriverEventExpectingClose, JournalReaderTraceEvent::DriverEventCompletedBlockRead, JournalReaderTraceEvent::DriverEventExpectedCloseGotClose, @@ -483,12 +633,13 @@ fn corruption_after_reopen() { JournalWriterTraceEvent::ReinitializeAttempt, JournalWriterTraceEvent::DriverEventAttemptCommit { event: DriverEventKind::Reopened, - event_id: repaired_last_event_id, - prev_id: repaired_last_event_id - 1 // close event + event_id: modified_journal_info.init.corrupted_event_id, + prev_id: modified_journal_info.init.corrupted_event_id - 1 // close event }, JournalWriterTraceEvent::DriverEventCompleted, JournalWriterTraceEvent::ReinitializeComplete - ] + ], + "failed at {trim_size} for journal {journal_id}" ) } // now close this so that this works with the post repair handler @@ -498,10 +649,16 @@ fn corruption_after_reopen() { } else { assert_eq!( open_result.unwrap_err().kind(), - &ErrorKind::IoError(IoErrorKind::UnexpectedEof.into()) + &ErrorKind::IoError(IoErrorKind::UnexpectedEof.into()), + "failed at {trim_size} for journal {journal_id}" ); - if repaired_last_event_id == 1 { + if modified_journal_info.init.last_executed_event_id == 1 { // empty log, only the reopen + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); assert_eq!( trace, intovec![ @@ -512,10 +669,16 @@ fn corruption_after_reopen() { JournalReaderTraceEvent::DriverEventCompletedBlockRead, JournalReaderTraceEvent::DriverEventExpectedCloseGotClose, JournalReaderTraceEvent::DriverEventExpectingReopenBlock, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id) - ] + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id + ) + ], + "failed at {trim_size} for journal {journal_id}" ); } else { + // we will have upto the last event since only the reopen is gone + assert_eq!(db.data().len(), POST_TRIALS_SIZE); + assert_eq!(*db.data().last().unwrap(), keyfmt(TRIALS - 1)); assert_eq!( &trace[trace.len() - 5..], intovec![ @@ -523,13 +686,16 @@ fn corruption_after_reopen() { JournalReaderTraceEvent::DriverEventCompletedBlockRead, JournalReaderTraceEvent::DriverEventExpectedCloseGotClose, JournalReaderTraceEvent::DriverEventExpectingReopenBlock, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id) - ] + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id + ) + ], + "failed at {trim_size} for journal {journal_id}" ); } } }, - |journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| { + |journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| { assert!(reopen_result.is_ok()); if trim_size == DriverEvent::FULL_EVENT_SIZE { // see earlier comment @@ -543,9 +709,20 @@ fn corruption_after_reopen() { repair_result.unwrap(), RepairResult::UnspecifiedLoss( (DriverEvent::FULL_EVENT_SIZE - trim_size) as u64 - ) + ), + "failed at {trim_size} for journal {journal_id}" ); } + if modified_journal_info.init.last_executed_event_id == 1 { + assert_eq!( + db.data().len(), + 0, + "failed at {trim_size} for journal {journal_id}" + ); + } else { + assert_eq!(db.data().len(), POST_TRIALS_SIZE); + assert_eq!(*db.data().last().unwrap(), keyfmt(TRIALS - 1)); + } let _ = debug_get_trace(); let _ = debug_get_offsets(); }, @@ -576,7 +753,7 @@ fn corruption_at_runtime() { let mut jrnl = create_journal(jrnl_id)?; sdb.push(&mut jrnl, KEY)?; // don't close - Ok(0) + Ok(InitializerInfo::new_last_event(0)) }, offset, ), @@ -586,19 +763,19 @@ fn corruption_at_runtime() { let mut op_count = 0; let mut sdb = SimpleDB::new(); let mut jrnl = create_journal(jrnl_id)?; - for _ in 0..TRIALS { + for _ in 1..=TRIALS { sdb.push(&mut jrnl, KEY)?; op_count += 1; } // don't close - Ok(op_count) + Ok(InitializerInfo::new_last_event(op_count)) }, offset, ), ]; emulate_final_event_corruption( initializers, - |journal_id, repaired_last_event_id, trim_size, _db, open_result, _modstat| { + |journal_id, modified_journal_info, trim_size, db, open_result| { let trace = debug_get_trace(); let err = open_result.unwrap_err(); assert_eq!( @@ -607,7 +784,12 @@ fn corruption_at_runtime() { "failed for journal {journal_id} with trim_size {trim_size}" ); if trim_size > offset - (sizeof!(u128) + sizeof!(u64)) { - if repaired_last_event_id == 0 { + if modified_journal_info.init.last_executed_event_id == 0 { + assert_eq!( + db.data().len(), + 0, + "failed for journal {journal_id} with trim_size {trim_size}" + ); assert_eq!( trace, intovec![ @@ -617,6 +799,17 @@ fn corruption_at_runtime() { "failed for journal {journal_id} with trim_size {trim_size}" ) } else { + // we lost the last server event, so we'll have one key less + assert_eq!( + db.data().len(), + TRIALS - 1, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + assert_eq!( + db.data()[TRIALS - 2], + KEY, + "failed for journal {journal_id} with trim_size {trim_size}" + ); assert_eq!( &trace[trace.len() - 4..], intovec![ @@ -629,8 +822,13 @@ fn corruption_at_runtime() { ) } } else { - if repaired_last_event_id == 0 { + if modified_journal_info.init.last_executed_event_id == 0 { // empty log + assert_eq!( + db.data().len(), + 0, + "failed for journal {journal_id} with trim_size {trim_size}" + ); assert_eq!( trace, intovec![ @@ -643,11 +841,24 @@ fn corruption_at_runtime() { "failed for journal {journal_id} with trim_size {trim_size}" ); } else { + // we lost the last server event, so we'll have one key less + assert_eq!( + db.data().len(), + TRIALS - 1, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + assert_eq!( + db.data()[TRIALS - 2], + KEY, + "failed for journal {journal_id} with trim_size {trim_size}" + ); assert_eq!( &trace[trace.len() - 4..], intovec![ JournalReaderTraceEvent::LookingForEvent, - JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1), + JournalReaderTraceEvent::AttemptingEvent( + modified_journal_info.init.corrupted_event_id - 1 + ), JournalReaderTraceEvent::DetectedServerEvent, JournalReaderTraceEvent::ServerEventMetadataParsed, ], @@ -656,13 +867,31 @@ fn corruption_at_runtime() { } } }, - |journal_id, _repaired_last_id, trim_size, repair_result, _db, reopen_result| { + |journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| { assert!(reopen_result.is_ok()); assert_eq!( repair_result.unwrap(), RepairResult::UnspecifiedLoss((offset - trim_size) as u64), "failed for journal {journal_id} with trim_size {trim_size}" ); + if modified_journal_info.init.last_executed_event_id == 0 { + assert_eq!( + db.data().len(), + 0, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + } else { + assert_eq!( + db.data().len(), + TRIALS - 1, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + assert_eq!( + db.data()[TRIALS - 2], + KEY, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + } let _ = debug_get_trace(); }, ) @@ -684,7 +913,7 @@ fn midway_corruption_close() { )?; RawJournalWriter::close_driver(&mut jrnl)?; drop(jrnl); - Ok(0) // close (to corrupt), reopen, close + Ok(InitializerInfo::new(0, 2)) // close (to corrupt), reopen, close }), Initializer::new_driver_type( "midway_corruption_close_events_before_second_close", @@ -692,7 +921,18 @@ fn midway_corruption_close() { { // create and close let mut jrnl = create_journal::(jrnl_id)?; - RawJournalWriter::close_driver(&mut jrnl)?; + RawJournalWriter::close_driver(&mut jrnl)?; // (0) + } + let op_cnt; + { + // reopen, apply mix and close + let mut jrnl = open_journal::( + jrnl_id, + &SimpleDB::new(), + JournalSettings::default(), + )?; // (1) + op_cnt = apply_event_mix(&mut jrnl)?; + RawJournalWriter::close_driver(&mut jrnl)?; // <-- (op_cnt + 2) corrupt this one } { // reopen and close @@ -700,69 +940,120 @@ fn midway_corruption_close() { jrnl_id, &SimpleDB::new(), JournalSettings::default(), - )?; - RawJournalWriter::close_driver(&mut jrnl)?; // <-- corrupt this one + )?; // (op_cnt + 3) + RawJournalWriter::close_driver(&mut jrnl)?; // (op_cnt + 4) + } + Ok(InitializerInfo::new(op_cnt + 2, op_cnt + 4)) + }, + ), + Initializer::new_driver_type( + "midway_corruption_close_events_before_third_close", + |jrnl_id| { + { + // create and close + let mut jrnl = create_journal::(jrnl_id)?; + RawJournalWriter::close_driver(&mut jrnl)?; // (0) } { + // reopen and close let mut jrnl = open_journal::( jrnl_id, &SimpleDB::new(), JournalSettings::default(), - )?; - let _ = apply_event_mix(&mut jrnl)?; - RawJournalWriter::close_driver(&mut jrnl)?; + )?; // (1) + RawJournalWriter::close_driver(&mut jrnl)?; // <-- (2) corrupt this one } - Ok(2) // corrupt the second close event + let op_cnt; + { + let mut jrnl = open_journal::( + jrnl_id, + &SimpleDB::new(), + JournalSettings::default(), + )?; // (3) + op_cnt = apply_event_mix(&mut jrnl)?; // (3 + op_count) + RawJournalWriter::close_driver(&mut jrnl)?; // (4 + op_count) + } + Ok(InitializerInfo::new(2, op_cnt + 4)) // corrupt the second close event }, ), ]; debug_set_offset_tracking(true); emulate_midway_corruption( initializers, - |journal_id, _last_id, trim_size, db, open_result, _modstat| { + |journal_id, modified_journal_info, trim_size, db, open_result| { assert!( open_result.is_err(), "failed for journal {journal_id} with trim_size {trim_size}" ); - // all data will be lost, so the DB will be empty - assert_eq!( - db.data().len(), - 0, - "failed for journal {journal_id} with trim_size {trim_size}" - ); + match modified_journal_info.initializer_id { + 0 | 2 => { + // in the first and third case, (0) no data is present (2) all data is lost + // all data will be lost, so the DB will be empty + assert_eq!( + db.data().len(), + 0, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + } + 1 => { + // in this case, all elements will be preserved + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); + } + _ => panic!(), + } let _ = debug_get_offsets(); let _ = debug_get_trace(); }, - |journal_id, last_id, trim_size, repair_result, db, reopen_result| { + |journal_id, modified_journal_info, trim_size, repair_result, db, reopen_result| { let _ = reopen_result.unwrap(); - // all data will be lost, so the DB will be empty - assert_eq!( - db.data().len(), - 0, - "failed for journal {journal_id} with trim_size {trim_size}" - ); - if last_id == 0 { - assert_eq!( - repair_result.unwrap(), - RepairResult::UnspecifiedLoss( - ((DriverEvent::FULL_EVENT_SIZE * 3) - trim_size) as u64 - ), - "failed for journal {journal_id} with trim_size {trim_size}" - ); - } else { - // this is a serious midway corruption with major data loss - let full_log_size = File::open(journal_id).unwrap().f_len().unwrap(); - assert_eq!( - repair_result.unwrap(), - RepairResult::UnspecifiedLoss( - full_log_size - - <::Spec as FileSpecV1>::SIZE // account for header - as u64 - - (DriverEvent::FULL_EVENT_SIZE * 2) as u64 // account for close (0), reopen(1) - - trim_size as u64 // account for trim - ), - "failed for journal {journal_id} with trim_size {trim_size}" - ); + match modified_journal_info.initializer_id { + 0 | 2 => { + // all data will be lost, so the DB will be empty + assert_eq!( + db.data().len(), + 0, + "failed for journal {journal_id} with trim_size {trim_size}" + ); + if modified_journal_info.init.corrupted_event_id == 0 + && modified_journal_info.init.not_last_event() + { + // the first event was corrupted + assert_eq!( + repair_result.unwrap(), + RepairResult::UnspecifiedLoss( + ((DriverEvent::FULL_EVENT_SIZE * 3) - trim_size) as u64 + ), + "failed for journal {journal_id} with trim_size {trim_size}" + ); + } else { + // this is a serious midway corruption with major data loss + let full_log_size = File::open(journal_id).unwrap().f_len().unwrap(); + assert_eq!( + repair_result.unwrap(), + RepairResult::UnspecifiedLoss( + full_log_size + - <::Spec as FileSpecV1>::SIZE // account for header + as u64 + - (DriverEvent::FULL_EVENT_SIZE * 2) as u64 // account for close (0), reopen(1) + - trim_size as u64 // account for trim + ), + "failed for journal {journal_id} with trim_size {trim_size}" + ); + } + } + 1 => { + // in this case, all elements will be preserved + assert_eq!( + *db.data().last().unwrap(), + keyfmt(TRIALS - 1), + "failed at {trim_size} for journal {journal_id}" + ); + } + _ => panic!(), } let _ = debug_get_trace(); let _ = debug_get_offsets();