diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs index 2968364b..0ebe0718 100644 --- a/server/src/engine/macros.rs +++ b/server/src/engine/macros.rs @@ -401,7 +401,7 @@ macro_rules! local { } macro_rules! local_mut { - ($ident:ident, $call:expr) => {{ + ($ident:expr, $call:expr) => {{ #[inline(always)] fn _f(v: &::std::cell::RefCell, f: impl FnOnce(&mut T) -> U) -> U { f(&mut *v.borrow_mut()) diff --git a/server/src/engine/storage/common/interface/vfs.rs b/server/src/engine/storage/common/interface/vfs.rs index 2f670de2..cfd77af7 100644 --- a/server/src/engine/storage/common/interface/vfs.rs +++ b/server/src/engine/storage/common/interface/vfs.rs @@ -50,6 +50,7 @@ pub mod vfs_utils { } local!( static RANDOM_WRITE_CRASH: WriteCrashKind = WriteCrashKind::None; + pub(super) static RNG: Option = None; ); /// WARNING: A random write crash automatically degrades to a [`WriteCrashKind::Zero`] as soon as it completes /// to prevent any further data writes (due to retries in @@ -204,9 +205,18 @@ impl VFile { Ok(bytes.len() as _) } vfs_utils::WriteCrashKind::Random => { + let actual_write_length = local_mut!(vfs_utils::RNG, |rng| { + match rng { + Some(ref mut rng) => test_utils::random_number(0, bytes.len(), rng), + None => { + let mut rng_ = rand::thread_rng(); + let r = test_utils::random_number(0, bytes.len(), &mut rng_); + *rng = Some(rng_); + r + } + } + }); // write some random part of the buffer into this file - let mut rng = rand::thread_rng(); - let actual_write_length = test_utils::random_number(0, bytes.len(), &mut rng); if self.pos + actual_write_length > self.data.len() { self.data.resize(self.pos + actual_write_length, 0); } diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs b/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs index 23cdcb9d..a85b749c 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs @@ -1531,6 +1531,7 @@ fn emulate_failure_for_rollback( post_rollback(&db); RawJournalWriter::close_driver(&mut jrnl).unwrap(); } + FileSystem::remove_file(journal_id).unwrap(); } #[test] @@ -1576,41 +1577,45 @@ fn rollback_write_zero_nonempty_log() { #[test] fn rollback_random_write_failure_empty_log() { - emulate_failure_for_rollback( - "rollback_random_write_failure_empty_log", - |db, jrnl| { - vfs_utils::debug_enable_random_write_crash(); - let r = db.push(jrnl, "hello, world"); - vfs_utils::debug_disable_write_crash(); - r - }, - |e| match e.kind() { - ErrorKind::IoError(io) if io.kind() == IoErrorKind::WriteZero => {} - unexpected => panic!("expected write zero, got {unexpected:?}"), - }, - |db| assert_eq!(db.data().len(), 0), - ); + for _ in 0..100 { + emulate_failure_for_rollback( + "rollback_random_write_failure_empty_log", + |db, jrnl| { + vfs_utils::debug_enable_random_write_crash(); + let r = db.push(jrnl, "hello, world"); + vfs_utils::debug_disable_write_crash(); + r + }, + |e| match e.kind() { + ErrorKind::IoError(io) if io.kind() == IoErrorKind::WriteZero => {} + unexpected => panic!("expected write zero, got {unexpected:?}"), + }, + |db| assert_eq!(db.data().len(), 0), + ); + } } #[test] fn rollback_random_write_failure_log() { - emulate_failure_for_rollback( - "rollback_random_write_failure_log", - |db, jrnl| { - // commit a single "good" event - db.push(jrnl, "my good key")?; - vfs_utils::debug_enable_random_write_crash(); - let r = db.push(jrnl, "this won't go in"); - vfs_utils::debug_disable_write_crash(); - r - }, - |e| match e.kind() { - ErrorKind::IoError(io) if io.kind() == IoErrorKind::WriteZero => {} - unexpected => panic!("expected write zero, got {unexpected:?}"), - }, - |db| { - assert_eq!(db.data().len(), 1); - assert_eq!(db.data()[0], "my good key") - }, - ) + for _ in 0..100 { + emulate_failure_for_rollback( + "rollback_random_write_failure_log", + |db, jrnl| { + // commit a single "good" event + db.push(jrnl, "my good key")?; + vfs_utils::debug_enable_random_write_crash(); + let r = db.push(jrnl, "this won't go in"); + vfs_utils::debug_disable_write_crash(); + r + }, + |e| match e.kind() { + ErrorKind::IoError(io) if io.kind() == IoErrorKind::WriteZero => {} + unexpected => panic!("expected write zero, got {unexpected:?}"), + }, + |db| { + assert_eq!(db.data().len(), 1); + assert_eq!(db.data()[0], "my good key") + }, + ) + } }