gvsafronov
/
skytable
1
0
Fork 0
Code Pull Requests Activity

storage: Test more runtime crash scenarios

Browse Source
next
Sayan Nandan 6 months ago
parent 267c9a6dd1
commit 8c21f129de
No known key found for this signature in database
GPG Key ID: 0EBD769024B24F0A
2 changed files with 110 additions and 55 deletions
Show Stats Download Patch File Download Diff File

4
server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs
Unescape Escape View File

@ -122,8 +122,8 @@ fn journal_open_close() {
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::AttemptingEvent(1),
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(1),
JournalReaderTraceEvent::DriverEventExpectingReopenGotReopen,
JournalReaderTraceEvent::ReopenSuccess,
// now read close event
@ -263,8 +263,8 @@ fn journal_with_server_single_event() {
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
// now read reopen event
JournalReaderTraceEvent::AttemptingEvent(2),
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(2),
JournalReaderTraceEvent::DriverEventExpectingReopenGotReopen,
JournalReaderTraceEvent::ReopenSuccess,
// now read close event

161
server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs
Unescape Escape View File

@ -122,6 +122,20 @@ fn emulate_corrupted_final_event(
}
}
fn apply_event_mix(jrnl: &mut RawJournalWriter<SimpleDBJournal>) -> RuntimeResult<u64> {
let mut op_count = 0;
let mut sdb = SimpleDB::new();
for num in 1..=100 {
op_count += 1;
sdb.push(jrnl, format!("key-{num}"))?;
if num % 10 == 0 {
op_count += 1;
sdb.pop(jrnl)?;
}
}
Ok(op_count)
}
#[test]
fn corruption_before_close() {
let initializers: Vec<Initializer> = vec![
@ -133,17 +147,8 @@ fn corruption_before_close() {
}),
// open, apply mix of events, close
("close_event_corruption.db", |jrnl_id| {
let mut operation_count = 0;
let mut sdb = SimpleDB::new();
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
for num in 1..=100 {
operation_count += 1;
sdb.push(&mut jrnl, format!("key-{num}"))?;
if num % 10 == 0 {
operation_count += 1;
sdb.pop(&mut jrnl)?;
}
}
let operation_count = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
Ok(operation_count)
}),
@ -241,14 +246,25 @@ fn corruption_before_close() {
#[test]
fn corruption_after_reopen() {
let initializers: Vec<Initializer> = vec![("corruption_after_reopen.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(1)
})];
let initializers: Vec<Initializer> = vec![
("corruption_after_reopen.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(1)
}),
("corruption_after_ropen_multi_before_close.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
let operation_count = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(operation_count + 1) // + 1 since we have the reopen event which is the next event that'll vanish
}),
];
emulate_corrupted_final_event(
initializers,
|journal_id, repaired_last_event_id, trim_size, open_result| {
@ -265,28 +281,53 @@ fn corruption_after_reopen() {
*/
let mut jrnl =
open_result.expect(&format!("failed at {trim_size} for journal {journal_id}"));
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: 0
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete,
],
"failed at {trim_size} for journal {journal_id}"
);
if repaired_last_event_id == 1 {
// empty log, only the reopen
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: 0
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete,
],
"failed at {trim_size} for journal {journal_id}"
);
} else {
assert_eq!(
&trace[trace.len() - 12..],
intovec![
JournalReaderTraceEvent::ServerEventAppliedSuccess,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1), // close event
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: repaired_last_event_id - 1 // close event
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete
]
)
}
// now close this so that this works with the post repair handler
RawJournalWriter::close_driver(&mut jrnl).unwrap();
let _ = obtain_offsets();
@ -296,19 +337,33 @@ fn corruption_after_reopen() {
open_result.unwrap_err().kind(),
&ErrorKind::IoError(IoErrorKind::UnexpectedEof.into())
);
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
if repaired_last_event_id == 1 {
// empty log, only the reopen
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
} else {
assert_eq!(
&trace[trace.len() - 5..],
intovec![
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
}
}
},
|journal_id, trim_size, repair_result, reopen_result| {

Write Preview
Loading…
Cancel
Save