From 0573e809925e3126b943f903d5afd4e224e8a37f Mon Sep 17 00:00:00 2001 From: Sayan Nandan Date: Thu, 29 Feb 2024 23:01:43 +0530 Subject: [PATCH] Add new fs abstraction --- server/src/engine/core/model/alt.rs | 67 +- server/src/engine/core/model/mod.rs | 98 ++- server/src/engine/core/space.rs | 109 ++- server/src/engine/core/tests/ddl_model/alt.rs | 8 +- server/src/engine/core/tests/ddl_model/crt.rs | 2 +- .../src/engine/core/tests/ddl_space/alter.rs | 10 +- .../src/engine/core/tests/ddl_space/create.rs | 8 +- server/src/engine/core/tests/dml/delete.rs | 4 +- server/src/engine/core/tests/dml/insert.rs | 6 +- server/src/engine/core/tests/dml/select.rs | 15 +- server/src/engine/core/tests/dml/update.rs | 12 +- server/src/engine/error.rs | 6 + server/src/engine/fractal/drivers.rs | 34 +- server/src/engine/fractal/mgr.rs | 4 +- server/src/engine/fractal/mod.rs | 25 +- server/src/engine/fractal/test_utils.rs | 43 +- server/src/engine/macros.rs | 14 +- .../src/engine/storage/common/interface/fs.rs | 568 ++++++++++++++ .../engine/storage/common/interface/fs_imp.rs | 205 ----- .../storage/common/interface/fs_traits.rs | 176 ----- .../engine/storage/common/interface/mod.rs | 5 +- .../common/interface/{fs_test.rs => vfs.rs} | 708 +++++++----------- .../storage/common/sdss/impls/sdss_r1/mod.rs | 23 +- .../storage/common/sdss/impls/sdss_r1/rw.rs | 212 ++---- .../r1/impls/gns/tests/full_chain.rs | 32 +- server/src/engine/storage/mod.rs | 12 +- server/src/engine/storage/v1/loader.rs | 37 +- server/src/engine/storage/v1/mod.rs | 10 +- .../engine/storage/v1/raw/batch_jrnl/mod.rs | 14 +- .../storage/v1/raw/batch_jrnl/persist.rs | 10 +- .../storage/v1/raw/batch_jrnl/restore.rs | 32 +- .../src/engine/storage/v1/raw/journal/raw.rs | 55 +- server/src/engine/storage/v1/raw/rw.rs | 141 ++-- server/src/engine/storage/v1/raw/sysdb.rs | 8 +- server/src/engine/storage/v2/impls/gns_log.rs | 9 +- .../engine/storage/v2/impls/mdl_journal.rs | 100 ++- .../storage/v2/impls/tests/model_driver.rs | 24 +- server/src/engine/storage/v2/mod.rs | 11 +- .../src/engine/storage/v2/raw/journal/mod.rs | 81 +- .../engine/storage/v2/raw/journal/raw/mod.rs | 74 +- .../storage/v2/raw/journal/raw/tests.rs | 48 +- .../engine/storage/v2/raw/journal/tests.rs | 44 +- 42 files changed, 1482 insertions(+), 1622 deletions(-) create mode 100644 server/src/engine/storage/common/interface/fs.rs delete mode 100644 server/src/engine/storage/common/interface/fs_imp.rs delete mode 100644 server/src/engine/storage/common/interface/fs_traits.rs rename server/src/engine/storage/common/interface/{fs_test.rs => vfs.rs} (50%) diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs index 483e215e..104e1664 100644 --- a/server/src/engine/core/model/alt.rs +++ b/server/src/engine/core/model/alt.rs @@ -266,18 +266,17 @@ impl Model { AlterAction::Ignore => {} AlterAction::Add(new_fields) => { // TODO(@ohsayan): this impacts lockdown duration; fix it - if G::FS_IS_NON_NULL { - // prepare txn - let txn = gns::model::AlterModelAddTxn::new( - ModelIDRef::new_ref(&space_name, &space, &model_name, model), - &new_fields, - ); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - } + + // prepare txn + let txn = gns::model::AlterModelAddTxn::new( + ModelIDRef::new_ref(&space_name, &space, &model_name, model), + &new_fields, + ); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; let mut mutator = model.model_mutator(); new_fields .stseq_ord_kv() @@ -287,36 +286,32 @@ impl Model { }); } AlterAction::Remove(removed) => { - if G::FS_IS_NON_NULL { - // prepare txn - let txn = gns::model::AlterModelRemoveTxn::new( - ModelIDRef::new_ref(&space_name, space, &model_name, model), - &removed, - ); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - } + // prepare txn + let txn = gns::model::AlterModelRemoveTxn::new( + ModelIDRef::new_ref(&space_name, space, &model_name, model), + &removed, + ); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; let mut mutator = model.model_mutator(); removed.iter().for_each(|field_id| { mutator.remove_field(field_id.as_str()); }); } AlterAction::Update(updated) => { - if G::FS_IS_NON_NULL { - // prepare txn - let txn = gns::model::AlterModelUpdateTxn::new( - ModelIDRef::new_ref(&space_name, space, &model_name, model), - &updated, - ); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - } + // prepare txn + let txn = gns::model::AlterModelUpdateTxn::new( + ModelIDRef::new_ref(&space_name, space, &model_name, model), + &updated, + ); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; let mut mutator = model.model_mutator(); updated.into_iter().for_each(|(field_id, field)| { mutator.update_field(field_id.as_ref(), field); diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs index 93d601c9..b0233d2f 100644 --- a/server/src/engine/core/model/mod.rs +++ b/server/src/engine/core/model/mod.rs @@ -278,36 +278,32 @@ impl Model { } } // since we've locked this down, no one else can parallely create another model in the same space (or remove) - if G::FS_IS_NON_NULL { - let mut txn_driver = global.gns_driver().lock(); - // prepare txn - let txn = gns::model::CreateModelTxn::new( - SpaceIDRef::new(&space_name, &space), - &model_name, - &model, - ); - // attempt to initialize driver - global.initialize_model_driver( - &space_name, - space.get_uuid(), - &model_name, - model.get_uuid(), - )?; - // commit txn - txn_driver.driver_context( - |drv| drv.commit_event(txn), - || { - global.taskmgr_post_standard_priority(Task::new( - GenericTask::delete_model_dir( - &space_name, - space.get_uuid(), - &model_name, - model.get_uuid(), - ), - )) - }, - )?; - } + let mut txn_driver = global.gns_driver().lock(); + // prepare txn + let txn = gns::model::CreateModelTxn::new( + SpaceIDRef::new(&space_name, &space), + &model_name, + &model, + ); + // attempt to initialize driver + global.initialize_model_driver( + &space_name, + space.get_uuid(), + &model_name, + model.get_uuid(), + )?; + // commit txn + txn_driver.driver_context( + |drv| drv.commit_event(txn), + || { + global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir( + &space_name, + space.get_uuid(), + &model_name, + model.get_uuid(), + ))) + }, + )?; // update global state let _ = space.models_mut().insert(model_name.into()); let _ = global @@ -347,28 +343,26 @@ impl Model { return Err(QueryError::QExecDdlNotEmpty); } // okay this is looking good for us - if G::FS_IS_NON_NULL { - // prepare txn - let txn = gns::model::DropModelTxn::new(ModelIDRef::new( - SpaceIDRef::new(&space_name, &space), - &model_name, - model.get_uuid(), - model.delta_state().schema_current_version().value_u64(), - )); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - // request cleanup - global.purge_model_driver( - space_name, - space.get_uuid(), - model_name, - model.get_uuid(), - false, - ); - } + // prepare txn + let txn = gns::model::DropModelTxn::new(ModelIDRef::new( + SpaceIDRef::new(&space_name, &space), + &model_name, + model.get_uuid(), + model.delta_state().schema_current_version().value_u64(), + )); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; + // request cleanup + global.purge_model_driver( + space_name, + space.get_uuid(), + model_name, + model.get_uuid(), + false, + ); // update global state let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name)); let _ = space.models_mut().remove(model_name); diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs index e70fd2cc..b5feca6f 100644 --- a/server/src/engine/core/space.rs +++ b/server/src/engine/core/space.rs @@ -24,7 +24,7 @@ * */ -use crate::engine::storage::safe_interfaces::paths_v1; +use crate::engine::storage::safe_interfaces::{paths_v1, FileSystem}; use super::EntityIDRef; @@ -35,7 +35,6 @@ use { fractal::{GenericTask, GlobalInstanceLike, Task}, idx::STIndex, ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace}, - storage::safe_interfaces::FSInterface, txn::{self, SpaceIDRef}, }, std::collections::HashSet, @@ -167,24 +166,20 @@ impl Space { } } // commit txn - if G::FS_IS_NON_NULL { - // prepare txn - let txn = txn::gns::space::CreateSpaceTxn::new(space.props(), &space_name, &space); - // try to create space for...the space - G::FileSystem::fs_create_dir_all(&paths_v1::space_dir( - &space_name, - space.get_uuid(), - ))?; - // commit txn - global.gns_driver().lock().driver_context( - |drv| drv.commit_event(txn), - || { - global.taskmgr_post_standard_priority(Task::new( - GenericTask::delete_space_dir(&space_name, space.get_uuid()), - )) - }, - )?; - } + // prepare txn + let txn = txn::gns::space::CreateSpaceTxn::new(space.props(), &space_name, &space); + // try to create space for...the space + FileSystem::create_dir_all(&paths_v1::space_dir(&space_name, space.get_uuid()))?; + // commit txn + global.gns_driver().lock().driver_context( + |drv| drv.commit_event(txn), + || { + global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir( + &space_name, + space.get_uuid(), + ))) + }, + )?; // update global state let _ = spaces.st_insert(space_name, space); if if_not_exists { @@ -214,19 +209,15 @@ impl Space { Some(patch) => patch, None => return Err(QueryError::QExecDdlInvalidProperties), }; - if G::FS_IS_NON_NULL { - // prepare txn - let txn = txn::gns::space::AlterSpaceTxn::new( - SpaceIDRef::new(&space_name, space), - &patch, - ); - // commit - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - } + // prepare txn + let txn = + txn::gns::space::AlterSpaceTxn::new(SpaceIDRef::new(&space_name, space), &patch); + // commit + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; // merge dict::rmerge_data_with_patch(space.props_mut(), patch); // the `env` key may have been popped, so put it back (setting `env: null` removes the env key and we don't want to waste time enforcing this in the @@ -255,20 +246,18 @@ impl Space { } }; // commit drop - if G::FS_IS_NON_NULL { - // prepare txn - let txn = - txn::gns::space::DropSpaceTxn::new(SpaceIDRef::new(&space_name, &space)); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - // request cleanup - global.taskmgr_post_standard_priority(Task::new( - GenericTask::delete_space_dir(&space_name, space.get_uuid()), - )); - } + // prepare txn + let txn = txn::gns::space::DropSpaceTxn::new(SpaceIDRef::new(&space_name, &space)); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; + // request cleanup + global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir( + &space_name, + space.get_uuid(), + ))); let space_uuid = space.get_uuid(); for model in space.models.into_iter() { let e: EntityIDRef<'static> = unsafe { @@ -305,20 +294,18 @@ impl Space { return Err(QueryError::QExecDdlNotEmpty); } // okay, it's empty; good riddance - if G::FS_IS_NON_NULL { - // prepare txn - let txn = - txn::gns::space::DropSpaceTxn::new(SpaceIDRef::new(&space_name, &space)); - // commit txn - global - .gns_driver() - .lock() - .driver_context(|drv| drv.commit_event(txn), || {})?; - // request cleanup - global.taskmgr_post_standard_priority(Task::new( - GenericTask::delete_space_dir(&space_name, space.get_uuid()), - )); - } + // prepare txn + let txn = txn::gns::space::DropSpaceTxn::new(SpaceIDRef::new(&space_name, &space)); + // commit txn + global + .gns_driver() + .lock() + .driver_context(|drv| drv.commit_event(txn), || {})?; + // request cleanup + global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir( + &space_name, + space.get_uuid(), + ))); let _ = spaces.st_delete(space_name.as_str()); if if_exists { Ok(Some(true)) diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs index 089cba00..83f0de73 100644 --- a/server/src/engine/core/tests/ddl_model/alt.rs +++ b/server/src/engine/core/tests/ddl_model/alt.rs @@ -358,7 +358,7 @@ mod exec { }; #[test] fn simple_add() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("simple_add"); super::exec_plan( &global, true, @@ -388,7 +388,7 @@ mod exec { } #[test] fn simple_remove() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("simple_remove"); super::exec_plan( &global, true, @@ -413,7 +413,7 @@ mod exec { } #[test] fn simple_update() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("simple_update"); super::exec_plan( &global, true, @@ -431,7 +431,7 @@ mod exec { } #[test] fn failing_alter_nullable_switch_need_lock() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("failing_alter_nullable_switch_need_lock"); assert_eq!( super::exec_plan( &global, diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs index d519a365..4956317f 100644 --- a/server/src/engine/core/tests/ddl_model/crt.rs +++ b/server/src/engine/core/tests/ddl_model/crt.rs @@ -146,7 +146,7 @@ mod exec { #[test] fn simple() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("exec_simple_create"); exec_create_new_space( &global, "create model myspace.mymodel(username: string, password: binary)", diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs index 8973c3ff..7a3f3932 100644 --- a/server/src/engine/core/tests/ddl_space/alter.rs +++ b/server/src/engine/core/tests/ddl_space/alter.rs @@ -33,7 +33,7 @@ use crate::engine::{ #[test] fn alter_add_prop_env_var() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("alter_add_prop_env_var"); super::exec_create_alter( &global, "create space myspace", @@ -53,7 +53,7 @@ fn alter_add_prop_env_var() { #[test] fn alter_update_prop_env_var() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("alter_update_prop_env_var"); let uuid = super::exec_create( &global, "create space myspace with { env: { MY_NEW_PROP: 100 } }", @@ -83,7 +83,7 @@ fn alter_update_prop_env_var() { #[test] fn alter_remove_prop_env_var() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("alter_remove_prop_env_var"); let uuid = super::exec_create( &global, "create space myspace with { env: { MY_NEW_PROP: 100 } }", @@ -113,7 +113,7 @@ fn alter_remove_prop_env_var() { #[test] fn alter_nx() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("alter_nx"); assert_eq!( super::exec_alter( &global, @@ -127,7 +127,7 @@ fn alter_nx() { #[test] fn alter_remove_all_env() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("alter_remove_all_env"); let uuid = super::exec_create( &global, "create space myspace with { env: { MY_NEW_PROP: 100 } }", diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs index 1010b5d4..3087def7 100644 --- a/server/src/engine/core/tests/ddl_space/create.rs +++ b/server/src/engine/core/tests/ddl_space/create.rs @@ -33,7 +33,7 @@ use crate::engine::{ #[test] fn exec_create_space_simple() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("exec_create_space_simple"); super::exec_create(&global, "create space myspace", |spc| { assert!(spc.models().is_empty()) }) @@ -42,7 +42,7 @@ fn exec_create_space_simple() { #[test] fn exec_create_space_with_env() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("exec_create_space_with_env"); super::exec_create( &global, r#" @@ -69,7 +69,7 @@ fn exec_create_space_with_env() { #[test] fn exec_create_space_with_bad_env_type() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("exec_create_space_with_bad_env_type"); assert_eq!( super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(), QueryError::QExecDdlInvalidProperties @@ -78,7 +78,7 @@ fn exec_create_space_with_bad_env_type() { #[test] fn exec_create_space_with_random_property() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id("exec_create_space_with_random_property"); assert_eq!( super::exec_create( &global, diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs index 017ba0ee..4176b235 100644 --- a/server/src/engine/core/tests/dml/delete.rs +++ b/server/src/engine/core/tests/dml/delete.rs @@ -28,7 +28,7 @@ use crate::engine::{error::QueryError, fractal::test_utils::TestGlobal}; #[test] fn simple_delete() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_delete_simple_delete"); super::exec_delete( &global, "create model myspace.mymodel(username: string, password: string)", @@ -41,7 +41,7 @@ fn simple_delete() { #[test] fn delete_nonexisting() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_delete_delete_nonexisting"); assert_eq!( super::exec_delete( &global, diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs index c1c99cbc..fff7e337 100644 --- a/server/src/engine/core/tests/dml/insert.rs +++ b/server/src/engine/core/tests/dml/insert.rs @@ -31,7 +31,7 @@ struct Tuple(Vec<(Box, Datacell)>); #[test] fn insert_simple() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_insert_simple"); super::exec_insert( &global, "create model myspace.mymodel(username: string, password: string)", @@ -46,7 +46,7 @@ fn insert_simple() { #[test] fn insert_with_null() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_insert_with_null"); super::exec_insert( &global, "create model myspace.mymodel(username: string, null useless_password: string, null useless_email: string, null useless_random_column: uint64)", @@ -69,7 +69,7 @@ fn insert_with_null() { #[test] fn insert_duplicate() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_insert_duplicate"); super::exec_insert( &global, "create model myspace.mymodel(username: string, password: string)", diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs index 1e693f43..5a120aa2 100644 --- a/server/src/engine/core/tests/dml/select.rs +++ b/server/src/engine/core/tests/dml/select.rs @@ -31,7 +31,7 @@ use { #[test] fn simple_select_wildcard() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_simple_select_wildcard"); assert_eq!( super::exec_select( &global, @@ -46,7 +46,7 @@ fn simple_select_wildcard() { #[test] fn simple_select_specified_same_order() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_simple_select_specified_same_order"); assert_eq!( super::exec_select( &global, @@ -61,7 +61,8 @@ fn simple_select_specified_same_order() { #[test] fn simple_select_specified_reversed_order() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = + TestGlobal::new_with_driver_id_instant_update("dml_select_simple_select_specified_reversed_order"); assert_eq!( super::exec_select( &global, @@ -76,7 +77,7 @@ fn simple_select_specified_reversed_order() { #[test] fn select_null() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_select_null"); assert_eq!( super::exec_select( &global, @@ -91,7 +92,7 @@ fn select_null() { #[test] fn select_nonexisting() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_select_nonexisting"); assert_eq!( super::exec_select( &global, @@ -110,7 +111,7 @@ fn select_nonexisting() { #[test] fn select_all_wildcard() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_select_all_wildcard"); let ret = super::exec_select_all( &global, "create model myspace.mymodel(username: string, password: string)", @@ -137,7 +138,7 @@ fn select_all_wildcard() { #[test] fn select_all_onefield() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_select_select_all_onefield"); let ret = super::exec_select_all( &global, "create model myspace.mymodel(username: string, password: string)", diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs index 577cf465..24b175ea 100644 --- a/server/src/engine/core/tests/dml/update.rs +++ b/server/src/engine/core/tests/dml/update.rs @@ -30,7 +30,7 @@ use crate::engine::{ #[test] fn simple() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_simple"); assert_eq!( super::exec_update( &global, @@ -49,7 +49,7 @@ fn simple() { #[test] fn with_null() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_with_null"); assert_eq!( super::exec_update( &global, @@ -66,7 +66,7 @@ fn with_null() { #[test] fn with_list() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_with_list"); assert_eq!( super::exec_update( &global, @@ -86,7 +86,7 @@ fn with_list() { #[test] fn fail_operation_on_null() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_fail_operation_on_null"); assert_eq!( super::exec_update( &global, @@ -106,7 +106,7 @@ fn fail_operation_on_null() { #[test] fn fail_unknown_fields() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_fail_unknown_fields"); assert_eq!( super::exec_update( &global, @@ -132,7 +132,7 @@ fn fail_unknown_fields() { #[test] fn fail_typedef_violation() { - let global = TestGlobal::new_with_tmp_nullfs_driver(); + let global = TestGlobal::new_with_driver_id_instant_update("dml_update_fail_typedef_violation"); assert_eq!( super::exec_update( &global, diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs index e728f18f..e7104c5f 100644 --- a/server/src/engine/error.rs +++ b/server/src/engine/error.rs @@ -97,6 +97,12 @@ pub enum QueryError { QExecNeedLock = 112, } +direct_from! { + QueryError[_] => { + std::io::Error as SysServerError, + } +} + impl From for QueryError { fn from(e: super::fractal::error::Error) -> Self { match e.kind() { diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs index 8c8f3595..d4218cd0 100644 --- a/server/src/engine/fractal/drivers.rs +++ b/server/src/engine/fractal/drivers.rs @@ -29,7 +29,7 @@ use { crate::{ engine::{ error::{QueryError, QueryResult, RuntimeResult}, - storage::{safe_interfaces::FSInterface, GNSDriver, ModelDriver}, + storage::{GNSDriver, ModelDriver}, }, util::compiler, }, @@ -38,13 +38,13 @@ use { }; /// GNS driver -pub struct FractalGNSDriver { +pub struct FractalGNSDriver { status: util::Status, - pub(super) txn_driver: GNSDriver, + pub(super) txn_driver: GNSDriver, } -impl FractalGNSDriver { - pub(super) fn new(txn_driver: GNSDriver) -> Self { +impl FractalGNSDriver { + pub(super) fn new(txn_driver: GNSDriver) -> Self { Self { status: util::Status::new_okay(), txn_driver: txn_driver, @@ -52,7 +52,7 @@ impl FractalGNSDriver { } pub fn driver_context( &mut self, - f: impl Fn(&mut GNSDriver) -> RuntimeResult, + f: impl Fn(&mut GNSDriver) -> RuntimeResult, on_failure: impl Fn(), ) -> QueryResult { if self.status.is_iffy() { @@ -70,23 +70,23 @@ impl FractalGNSDriver { } } -pub struct ModelDrivers { - drivers: RwLock>>, +pub struct ModelDrivers { + drivers: RwLock>, } -impl ModelDrivers { +impl ModelDrivers { pub fn empty() -> Self { Self { drivers: RwLock::new(HashMap::new()), } } - pub fn drivers(&self) -> &RwLock>> { + pub fn drivers(&self) -> &RwLock> { &self.drivers } pub fn count(&self) -> usize { self.drivers.read().len() } - pub fn add_driver(&self, id: ModelUniqueID, batch_driver: ModelDriver) { + pub fn add_driver(&self, id: ModelUniqueID, batch_driver: ModelDriver) { assert!(self .drivers .write() @@ -96,19 +96,19 @@ impl ModelDrivers { pub fn remove_driver(&self, id: ModelUniqueID) { assert!(self.drivers.write().remove(&id).is_some()) } - pub fn into_inner(self) -> HashMap> { + pub fn into_inner(self) -> HashMap { self.drivers.into_inner() } } /// Model driver -pub struct FractalModelDriver { +pub struct FractalModelDriver { status: Arc, - batch_driver: Mutex>, + batch_driver: Mutex, } -impl FractalModelDriver { - pub(in crate::engine::fractal) fn init(batch_driver: ModelDriver) -> Self { +impl FractalModelDriver { + pub(in crate::engine::fractal) fn init(batch_driver: ModelDriver) -> Self { Self { status: Arc::new(util::Status::new_okay()), batch_driver: Mutex::new(batch_driver), @@ -118,7 +118,7 @@ impl FractalModelDriver { &self.status } /// Returns a reference to the batch persist driver - pub fn batch_driver(&self) -> &Mutex> { + pub fn batch_driver(&self) -> &Mutex { &self.batch_driver } pub fn close(self) -> RuntimeResult<()> { diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs index 9063a7a7..502a208c 100644 --- a/server/src/engine/fractal/mgr.rs +++ b/server/src/engine/fractal/mgr.rs @@ -35,7 +35,7 @@ use { data::uuid::Uuid, error::ErrorKind, storage::{ - safe_interfaces::{paths_v1, LocalFS, StdModelBatch}, + safe_interfaces::{paths_v1, StdModelBatch}, BatchStats, }, }, @@ -442,7 +442,7 @@ impl FractalMgr { fn try_write_model_data_batch( model: &Model, observed_size: usize, - mdl_driver: &super::drivers::FractalModelDriver, + mdl_driver: &super::drivers::FractalModelDriver, ) -> Result<(), (super::error::Error, BatchStats)> { if mdl_driver.status().is_iffy() { // don't mess this up any further diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs index b9cdb465..46c874e7 100644 --- a/server/src/engine/fractal/mod.rs +++ b/server/src/engine/fractal/mod.rs @@ -29,7 +29,7 @@ use { core::{dml::QueryExecMeta, model::Model, GlobalNS}, data::uuid::Uuid, storage::{ - safe_interfaces::{paths_v1, FSInterface, LocalFS}, + safe_interfaces::{paths_v1, FileSystem}, GNSDriver, ModelDriver, }, }, @@ -70,8 +70,8 @@ pub struct GlobalStateStart { /// Must be called iff this is the only thread calling it pub unsafe fn load_and_enable_all( gns: GlobalNS, - gns_driver: GNSDriver, - model_drivers: ModelDrivers, + gns_driver: GNSDriver, + model_drivers: ModelDrivers, ) -> GlobalStateStart { let model_cnt_on_boot = model_drivers.count(); let gns_driver = drivers::FractalGNSDriver::new(gns_driver); @@ -97,13 +97,11 @@ pub unsafe fn load_and_enable_all( /// Something that represents the global state pub trait GlobalInstanceLike { - type FileSystem: FSInterface; - const FS_IS_NON_NULL: bool = Self::FileSystem::NOT_NULL; // stat fn get_max_delta_size(&self) -> usize; // global namespace fn state(&self) -> &GlobalNS; - fn gns_driver(&self) -> &Mutex>; + fn gns_driver(&self) -> &Mutex; // model drivers fn initialize_model_driver( &self, @@ -148,12 +146,11 @@ pub trait GlobalInstanceLike { } impl GlobalInstanceLike for Global { - type FileSystem = LocalFS; // ns fn state(&self) -> &GlobalNS { self._namespace() } - fn gns_driver(&self) -> &Mutex> { + fn gns_driver(&self) -> &Mutex { &self.get_state().gns_driver } // taskmgr @@ -192,7 +189,7 @@ impl GlobalInstanceLike for Global { model_uuid: Uuid, ) -> RuntimeResult<()> { // create dir - LocalFS::fs_create_dir(&paths_v1::model_dir( + FileSystem::create_dir(&paths_v1::model_dir( space_name, space_uuid, model_name, model_uuid, ))?; // init driver @@ -271,16 +268,16 @@ impl Global { /// The global state struct GlobalState { gns: GlobalNS, - gns_driver: Mutex>, - mdl_driver: ModelDrivers, + gns_driver: Mutex, + mdl_driver: ModelDrivers, task_mgr: mgr::FractalMgr, } impl GlobalState { fn new( gns: GlobalNS, - gns_driver: drivers::FractalGNSDriver, - mdl_driver: ModelDrivers, + gns_driver: drivers::FractalGNSDriver, + mdl_driver: ModelDrivers, task_mgr: mgr::FractalMgr, ) -> Self { Self { @@ -290,7 +287,7 @@ impl GlobalState { task_mgr, } } - pub(self) fn get_mdl_drivers(&self) -> &ModelDrivers { + pub(self) fn get_mdl_drivers(&self) -> &ModelDrivers { &self.mdl_driver } pub(self) fn fractal_mgr(&self) -> &mgr::FractalMgr { diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs index 2d9d89ae..63fd4c99 100644 --- a/server/src/engine/fractal/test_utils.rs +++ b/server/src/engine/fractal/test_utils.rs @@ -35,7 +35,7 @@ use { error::ErrorKind, fractal::drivers::FractalModelDriver, storage::{ - safe_interfaces::{paths_v1, FSInterface, NullFS, StdModelBatch, VirtualFS}, + safe_interfaces::{paths_v1, FileSystem, StdModelBatch}, BatchStats, GNSDriver, ModelDriver, }, RuntimeResult, @@ -45,18 +45,18 @@ use { }; /// A `test` mode global implementation -pub struct TestGlobal { +pub struct TestGlobal { gns: GlobalNS, lp_queue: RwLock>>, #[allow(unused)] max_delta_size: usize, - txn_driver: Mutex>, - model_drivers: RwLock>>, + txn_driver: Mutex, + model_drivers: RwLock>, max_data_pressure: usize, } -impl TestGlobal { - fn new(gns: GlobalNS, max_delta_size: usize, txn_driver: GNSDriver) -> Self { +impl TestGlobal { + fn new(gns: GlobalNS, max_delta_size: usize, txn_driver: GNSDriver) -> Self { Self { gns, lp_queue: RwLock::default(), @@ -96,7 +96,12 @@ impl TestGlobal { } } -impl TestGlobal { +impl TestGlobal { + pub fn new_with_driver_id_instant_update(log_name: &str) -> Self { + let mut me = Self::new_with_driver_id(log_name); + me.set_max_data_pressure(1); + me + } pub fn new_with_driver_id(log_name: &str) -> Self { let gns = GlobalNS::empty(); let driver = match GNSDriver::create_gns_with_name(log_name) { @@ -116,27 +121,11 @@ impl TestGlobal { } } -impl TestGlobal { - pub fn new_with_vfs_driver(log_name: &str) -> Self { - Self::new_with_driver_id(log_name) - } -} - -impl TestGlobal { - pub fn new_with_nullfs_driver(log_name: &str) -> Self { - Self::new_with_driver_id(log_name) - } - pub fn new_with_tmp_nullfs_driver() -> Self { - Self::new_with_nullfs_driver("") - } -} - -impl GlobalInstanceLike for TestGlobal { - type FileSystem = Fs; +impl GlobalInstanceLike for TestGlobal { fn state(&self) -> &GlobalNS { &self.gns } - fn gns_driver(&self) -> &Mutex> { + fn gns_driver(&self) -> &Mutex { &self.txn_driver } fn taskmgr_post_high_priority(&self, task: Task) { @@ -190,7 +179,7 @@ impl GlobalInstanceLike for TestGlobal { model_uuid: Uuid, ) -> crate::engine::error::RuntimeResult<()> { // create model dir - Fs::fs_create_dir(&paths_v1::model_dir( + FileSystem::create_dir_all(&paths_v1::model_dir( space_name, space_uuid, model_name, model_uuid, ))?; let driver = ModelDriver::create_model_driver(&paths_v1::model_path( @@ -204,7 +193,7 @@ impl GlobalInstanceLike for TestGlobal { } } -impl Drop for TestGlobal { +impl Drop for TestGlobal { fn drop(&mut self) { let mut txn_driver = self.txn_driver.lock(); GNSDriver::close_driver(&mut txn_driver.txn_driver).unwrap(); diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs index eb36d00a..ee3090bc 100644 --- a/server/src/engine/macros.rs +++ b/server/src/engine/macros.rs @@ -410,7 +410,6 @@ macro_rules! local_mut { }}; } -#[cfg(test)] macro_rules! local_ref { ($ident:ident, $call:expr) => {{ #[inline(always)] @@ -448,3 +447,16 @@ macro_rules! array { $(#[$attr])*$vis const$name:[$ty;$name::LEN]=[$($expr),*];)* } } + +macro_rules! e { + ($e:expr) => {{ + #[inline(always)] + fn r(r: Result) -> Result + where + E2: From, + { + r.map_err(::core::convert::From::from) + } + r($e) + }}; +} diff --git a/server/src/engine/storage/common/interface/fs.rs b/server/src/engine/storage/common/interface/fs.rs new file mode 100644 index 00000000..07ce3622 --- /dev/null +++ b/server/src/engine/storage/common/interface/fs.rs @@ -0,0 +1,568 @@ +/* + * Created on Thu Feb 29 2024 + * + * This file is a part of Skytable + * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source + * NoSQL database written by Sayan Nandan ("the Author") with the + * vision to provide flexibility in data modelling without compromising + * on performance, queryability or scalability. + * + * Copyright (c) 2024, Sayan Nandan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * +*/ + +#![allow(dead_code)] + +/* + file system +*/ + +use std::io::BufWriter; + +#[cfg(test)] +use super::vfs::{VFileDescriptor, VirtualFS}; +use { + crate::IoResult, + std::{ + fs as std_fs, + io::{BufReader, Error, ErrorKind, Read, Seek, SeekFrom, Write}, + }, +}; + +pub struct FileSystem {} + +impl Default for FileSystem { + fn default() -> Self { + Self::instance() + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum FSContext { + Local, + Virtual, +} + +impl FileSystem { + pub fn instance() -> Self { + Self {} + } + fn context() -> FSContext { + local! { static CTX: FSContext = FSContext::Virtual; } + local_ref!(CTX, |ctx| *ctx) + } +} + +impl FileSystem { + #[inline(always)] + pub fn read(path: &str) -> IoResult> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().read().get_data(path), + } + } + std_fs::read(path) + } + #[inline(always)] + pub fn create_dir(path: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_create_dir(path), + } + } + std_fs::create_dir(path) + } + #[inline(always)] + pub fn create_dir_all(path: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_create_dir_all(path), + } + } + std_fs::create_dir_all(path) + } + #[inline(always)] + pub fn remove_dir(path: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_delete_dir(path), + } + } + std_fs::remove_dir(path) + } + #[inline(always)] + pub fn remove_dir_all(path: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_delete_dir_all(path), + } + } + std_fs::remove_dir_all(path) + } + #[inline(always)] + pub fn remove_file(path: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_remove_file(path), + } + } + std_fs::remove_file(path) + } + #[inline(always)] + pub fn rename(from: &str, to: &str) -> IoResult<()> { + #[cfg(test)] + { + match Self::context() { + FSContext::Local => {} + FSContext::Virtual => return VirtualFS::instance().write().fs_rename(from, to), + } + } + std_fs::rename(from, to) + } +} + +/* + file traits +*/ + +pub trait FileRead { + fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()>; + fn fread_exact_block(&mut self) -> IoResult<[u8; N]> { + let mut blk = [0; N]; + self.fread_exact(&mut blk).map(|_| blk) + } +} + +pub trait FileWrite { + fn fwrite(&mut self, buf: &[u8]) -> IoResult; + fn fwrite_all(&mut self, buf: &[u8]) -> IoResult<()> { + self.fwrite_all_count(buf).1 + } + fn fwrite_all_count(&mut self, buf: &[u8]) -> (u64, IoResult<()>) { + let len = buf.len() as u64; + let mut written = 0; + while written != len { + match self.fwrite(buf) { + Ok(0) => { + return ( + written, + Err(Error::new( + ErrorKind::WriteZero, + format!("could only write {} of {} bytes", written, buf.len()), + ) + .into()), + ) + } + Ok(n) => written += n, + Err(e) => return (written, Err(e)), + } + } + (written, Ok(())) + } +} + +pub trait FileWriteExt { + fn fsync_all(&mut self) -> IoResult<()>; + fn fsync_data(&mut self) -> IoResult<()>; + fn f_truncate(&mut self, new_size: u64) -> IoResult<()>; +} + +pub trait FileExt { + fn f_len(&self) -> IoResult; + fn f_cursor(&mut self) -> IoResult; + fn f_seek_start(&mut self, offset: u64) -> IoResult<()>; +} + +/* + file impls +*/ + +impl FileWrite for File { + fn fwrite(&mut self, buf: &[u8]) -> IoResult { + self.f.fwrite(buf) + } +} + +impl FileRead for File { + fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> { + self.f.fread_exact(buf) + } +} + +impl FileWriteExt for File { + fn fsync_all(&mut self) -> IoResult<()> { + self.f.fsync_all() + } + fn fsync_data(&mut self) -> IoResult<()> { + self.f.fsync_data() + } + fn f_truncate(&mut self, new_size: u64) -> IoResult<()> { + self.f.f_truncate(new_size) + } +} + +impl FileExt for File { + fn f_len(&self) -> IoResult { + self.f.f_len() + } + fn f_cursor(&mut self) -> IoResult { + self.f.f_cursor() + } + fn f_seek_start(&mut self, offset: u64) -> IoResult<()> { + self.f.f_seek_start(offset) + } +} + +/* + impls for local file +*/ + +trait LocalFile { + fn _mut(&mut self) -> &mut std_fs::File; + fn _ref(&self) -> &std_fs::File; +} + +impl LocalFile for BufReader { + fn _mut(&mut self) -> &mut std_fs::File { + self.get_mut() + } + fn _ref(&self) -> &std_fs::File { + self.get_ref() + } +} + +impl LocalFile for std_fs::File { + fn _mut(&mut self) -> &mut std_fs::File { + self + } + fn _ref(&self) -> &std_fs::File { + self + } +} + +impl FileWrite for W { + fn fwrite(&mut self, buf: &[u8]) -> IoResult { + self.write(buf).map(|x| x as u64) + } +} + +impl FileRead for R { + fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> { + self.read_exact(buf) + } +} + +impl FileWriteExt for Lf { + fn fsync_all(&mut self) -> IoResult<()> { + self._mut().sync_all() + } + fn fsync_data(&mut self) -> IoResult<()> { + self._mut().sync_data() + } + fn f_truncate(&mut self, new_size: u64) -> IoResult<()> { + self._mut().set_len(new_size) + } +} + +impl FileExt for Lf { + fn f_len(&self) -> IoResult { + self._ref().metadata().map(|md| md.len()) + } + fn f_cursor(&mut self) -> IoResult { + self._mut().stream_position() + } + fn f_seek_start(&mut self, offset: u64) -> IoResult<()> { + self._mut().seek(SeekFrom::Start(offset)).map(|_| ()) + } +} + +/* + impls for vfile +*/ + +#[cfg(test)] +impl FileWrite for AnyFile { + fn fwrite(&mut self, buf: &[u8]) -> IoResult { + match self { + Self::Local(lf) => lf.fwrite(buf), + Self::Virtual(vf) => vf.get_mut(&mut VirtualFS::instance().write()).fwrite(buf), + } + } +} + +#[cfg(test)] +impl FileRead for AnyFile { + fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> { + match self { + Self::Local(lf) => lf.fread_exact(buf), + Self::Virtual(vf) => vf + .get_mut(&mut VirtualFS::instance().write()) + .fread_exact(buf), + } + } +} + +#[cfg(test)] +impl FileWriteExt for AnyFile { + fn fsync_all(&mut self) -> IoResult<()> { + match self { + Self::Local(lf) => lf.fsync_all(), + Self::Virtual(_) => Ok(()), + } + } + fn fsync_data(&mut self) -> IoResult<()> { + match self { + Self::Local(lf) => lf.fsync_data(), + Self::Virtual(_) => Ok(()), + } + } + fn f_truncate(&mut self, new_size: u64) -> IoResult<()> { + match self { + Self::Local(lf) => lf.f_truncate(new_size), + Self::Virtual(vf) => vf + .get_mut(&mut VirtualFS::instance().write()) + .truncate(new_size), + } + } +} + +#[cfg(test)] +impl FileExt for AnyFile { + fn f_len(&self) -> IoResult { + match self { + Self::Local(lf) => lf.f_len(), + Self::Virtual(vf) => vf.get_ref(&VirtualFS::instance().read()).length(), + } + } + fn f_cursor(&mut self) -> IoResult { + match self { + Self::Local(lf) => lf.f_cursor(), + Self::Virtual(vf) => vf.get_ref(&VirtualFS::instance().read()).cursor(), + } + } + fn f_seek_start(&mut self, offset: u64) -> IoResult<()> { + match self { + Self::Local(lf) => lf.f_seek_start(offset), + Self::Virtual(vf) => vf + .get_mut(&mut VirtualFS::instance().write()) + .seek_from_start(offset), + } + } +} + +/* + file abstraction +*/ + +#[cfg(test)] +enum AnyFile { + Local(Lf), + Virtual(VFileDescriptor), +} + +pub struct File { + #[cfg(test)] + f: AnyFile, + #[cfg(not(test))] + f: std_fs::File, +} + +impl File { + pub fn open(path: &str) -> IoResult { + #[cfg(test)] + { + match FileSystem::context() { + FSContext::Local => {} + FSContext::Virtual => { + return VirtualFS::instance() + .write() + .fs_fopen_rw(path) + .map(|f| Self { + f: AnyFile::Virtual(f), + }) + } + } + } + let file = std_fs::File::options().read(true).write(true).open(path)?; + Ok(Self { + #[cfg(test)] + f: AnyFile::Local(file), + #[cfg(not(test))] + f: file, + }) + } + pub fn create(path: &str) -> IoResult { + #[cfg(test)] + { + match FileSystem::context() { + FSContext::Local => {} + FSContext::Virtual => { + return VirtualFS::instance() + .write() + .fs_fcreate_rw(path) + .map(|f| Self { + f: AnyFile::Virtual(f), + }) + } + } + } + let file = std_fs::File::options() + .create_new(true) + .read(true) + .write(true) + .open(path)?; + Ok(Self { + #[cfg(test)] + f: AnyFile::Local(file), + #[cfg(not(test))] + f: file, + }) + } + pub fn into_buffered_reader(self) -> BufferedReader { + BufferedReader::new(self.f) + } + pub fn into_buffered_writer(self) -> BufferedWriter { + BufferedWriter::new(self.f) + } +} + +/* + buffered readers and writers +*/ + +pub struct BufferedReader { + #[cfg(test)] + f: AnyFile>, + #[cfg(not(test))] + f: BufReader, +} + +impl BufferedReader { + fn new(#[cfg(test)] f: AnyFile, #[cfg(not(test))] f: std_fs::File) -> Self { + Self { + #[cfg(test)] + f: match f { + AnyFile::Local(lf) => AnyFile::Local(BufReader::new(lf)), + AnyFile::Virtual(vf) => AnyFile::Virtual(vf), + }, + #[cfg(not(test))] + f: BufReader::new(f), + } + } + pub fn into_inner(self) -> File { + File { + #[cfg(test)] + f: match self.f { + AnyFile::Local(lf) => AnyFile::Local(lf.into_inner()), + AnyFile::Virtual(vf) => AnyFile::Virtual(vf), + }, + #[cfg(not(test))] + f: self.f.into_inner(), + } + } +} + +impl FileRead for BufferedReader { + fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> { + self.f.fread_exact(buf) + } +} + +impl FileExt for BufferedReader { + fn f_len(&self) -> IoResult { + self.f.f_len() + } + fn f_cursor(&mut self) -> IoResult { + self.f.f_cursor() + } + fn f_seek_start(&mut self, offset: u64) -> IoResult<()> { + self.f.f_seek_start(offset) + } +} + +pub struct BufferedWriter { + #[cfg(test)] + f: AnyFile>, + #[cfg(not(test))] + f: BufWriter, +} + +impl BufferedWriter { + pub fn into_inner(self) -> IoResult { + let mut local; + #[cfg(test)] + { + match self.f { + AnyFile::Local(lf) => local = lf, + AnyFile::Virtual(vf) => { + return Ok(File { + f: AnyFile::Virtual(vf), + }) + } + } + } + #[cfg(not(test))] + { + local = self.f; + } + local.flush()?; + let local = local.into_inner().unwrap(); + Ok(File { + #[cfg(test)] + f: AnyFile::Local(local), + #[cfg(not(test))] + f: local, + }) + } + pub fn flush(&mut self) -> IoResult<()> { + let local; + #[cfg(test)] + { + match self.f { + AnyFile::Local(ref mut l) => local = l, + AnyFile::Virtual(_) => return Ok(()), + } + } + #[cfg(not(test))] + { + local = &mut self.f; + } + local.flush() + } + fn new(#[cfg(test)] f: AnyFile, #[cfg(not(test))] f: std_fs::File) -> Self { + Self { + #[cfg(test)] + f: match f { + AnyFile::Local(lf) => AnyFile::Local(BufWriter::new(lf)), + AnyFile::Virtual(vf) => AnyFile::Virtual(vf), + }, + #[cfg(not(test))] + f: BufWriter::new(f), + } + } +} diff --git a/server/src/engine/storage/common/interface/fs_imp.rs b/server/src/engine/storage/common/interface/fs_imp.rs deleted file mode 100644 index 4e56785f..00000000 --- a/server/src/engine/storage/common/interface/fs_imp.rs +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Created on Sun Jan 07 2024 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2024, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - super::fs_traits::{ - FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, FileInterfaceRead, - FileInterfaceWrite, FileInterfaceWriteExt, FileOpen, - }, - crate::engine::RuntimeResult, - std::{ - fs::{self, File}, - io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}, - }, -}; - -/* - local fs impls -*/ - -/// A type representing the host's local filesystem (or atleast where our data directory is) -pub struct LocalFS; - -fn cvt>(r: Result) -> Result { - r.map_err(Into::into) -} - -impl FSInterface for LocalFS { - type File = File; - fn fs_remove_file(fpath: &str) -> RuntimeResult<()> { - cvt(fs::remove_file(fpath)) - } - fn fs_rename(from: &str, to: &str) -> RuntimeResult<()> { - cvt(fs::rename(from, to)) - } - fn fs_create_dir(fpath: &str) -> RuntimeResult<()> { - cvt(fs::create_dir(fpath)) - } - fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> { - cvt(fs::create_dir_all(fpath)) - } - fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> { - cvt(fs::remove_dir(fpath)) - } - fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> { - cvt(fs::remove_dir_all(fpath)) - } - fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult> { - let r = || -> Result<_, std::io::Error> { - let f = File::options() - .create(true) - .read(true) - .write(true) - .open(fpath)?; - let md = f.metadata()?; - if md.len() == 0 { - Ok(FileOpen::Created(f)) - } else { - Ok(FileOpen::Existing(f)) - } - }; - cvt(r()) - } - fn fs_fopen_rw(fpath: &str) -> RuntimeResult { - let f = File::options().read(true).write(true).open(fpath)?; - Ok(f) - } - fn fs_fcreate_rw(fpath: &str) -> RuntimeResult { - let f = File::options() - .create_new(true) - .read(true) - .write(true) - .open(fpath)?; - Ok(f) - } -} - -/* - common impls for files -*/ - -impl FileInterfaceRead for R { - fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { - cvt(self.read_exact(buf)) - } -} - -impl FileInterfaceWrite for W { - fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult { - cvt(self.write(buf).map(|v| v as _)) - } -} - -/* - local file impls -*/ - -impl FileInterface for File { - type BufReader = BufReader; - type BufWriter = BufWriter; - fn upgrade_to_buffered_reader(self) -> RuntimeResult { - Ok(BufReader::new(self)) - } - fn upgrade_to_buffered_writer(self) -> RuntimeResult { - Ok(BufWriter::new(self)) - } - fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { - Ok(r.into_inner()) - } - fn downgrade_writer(mut r: Self::BufWriter) -> RuntimeResult { - // TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this? - r.flush()?; - let (me, err) = r.into_parts(); - match err { - Ok(x) if x.is_empty() => Ok(me), - Ok(_) | Err(_) => { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "failed to flush data from buffer into sink", - ) - .into()) - } - } - } -} - -/// A trait for handling wrappers of [`std::fs::File`] -trait AsLocalFile { - fn file(&self) -> &File; - fn file_mut(&mut self) -> &mut File; -} - -impl AsLocalFile for File { - fn file(&self) -> &File { - self - } - fn file_mut(&mut self) -> &mut File { - self - } -} - -impl AsLocalFile for BufReader { - fn file(&self) -> &File { - self.get_ref() - } - fn file_mut(&mut self) -> &mut File { - self.get_mut() - } -} - -impl AsLocalFile for BufWriter { - fn file(&self) -> &File { - self.get_ref() - } - fn file_mut(&mut self) -> &mut File { - self.get_mut() - } -} - -impl FileInterfaceBufWrite for BufWriter { - fn sync_write_cache(&mut self) -> RuntimeResult<()> { - // TODO(@ohsayan): maybe we'll want to explicitly handle not syncing this? - cvt(self.flush()) - } -} - -impl FileInterfaceExt for F { - fn fext_length(&self) -> RuntimeResult { - Ok(self.file().metadata()?.len()) - } - fn fext_cursor(&mut self) -> RuntimeResult { - cvt(self.file_mut().stream_position()) - } - fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> { - cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ())) - } -} - -impl FileInterfaceWriteExt for File { - fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> { - cvt(self.set_len(to)) - } -} diff --git a/server/src/engine/storage/common/interface/fs_traits.rs b/server/src/engine/storage/common/interface/fs_traits.rs deleted file mode 100644 index b184e5b8..00000000 --- a/server/src/engine/storage/common/interface/fs_traits.rs +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Created on Sun Jan 07 2024 - * - * This file is a part of Skytable - * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source - * NoSQL database written by Sayan Nandan ("the Author") with the - * vision to provide flexibility in data modelling without compromising - * on performance, queryability or scalability. - * - * Copyright (c) 2024, Sayan Nandan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * -*/ - -use { - crate::engine::RuntimeResult, - std::io::{Error as IoError, ErrorKind as IoErrorKind}, -}; - -#[derive(Debug, PartialEq)] -/// Result of opening a file -/// - Created: newly created file -/// - Existing: existing file that was reopened -pub enum FileOpen { - /// new file - Created(CF), - /// existing file - Existing(EF), -} - -pub trait FSInterface { - // settings - /// set to false if the file system is a special device like `/dev/null` - const NOT_NULL: bool = true; - // types - /// the file type that is returned by this file system - type File: FileInterface; - // functions - /// Remove a file - fn fs_remove_file(fpath: &str) -> RuntimeResult<()>; - /// Rename a file - fn fs_rename(from: &str, to: &str) -> RuntimeResult<()>; - /// Create a directory - fn fs_create_dir(fpath: &str) -> RuntimeResult<()>; - /// Create a directory and all corresponding path components - fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()>; - /// Delete a directory - fn fs_delete_dir(fpath: &str) -> RuntimeResult<()>; - /// Delete a directory and recursively remove all (if any) children - fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()>; - /// Open or create a file in R/W mode - /// - /// This will: - /// - Create a file if it doesn't exist - /// - Open a file it it does exist - fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult>; - /// Open an existing file - fn fs_fopen_rw(fpath: &str) -> RuntimeResult; - /// Create a new file - fn fs_fcreate_rw(fpath: &str) -> RuntimeResult; -} - -/// File interface definition -pub trait FileInterface: - FileInterfaceRead + FileInterfaceWrite + FileInterfaceWriteExt + FileInterfaceExt + Sized -{ - /// A buffered reader implementation - type BufReader: FileInterfaceRead + FileInterfaceExt; - /// A buffered writer implementation - type BufWriter: FileInterfaceBufWrite; - /// Get a buffered reader for this file - fn upgrade_to_buffered_reader(self) -> RuntimeResult; - /// Get a buffered writer for this file - fn upgrade_to_buffered_writer(self) -> RuntimeResult; - /// Get the file back from the buffered reader - fn downgrade_reader(r: Self::BufReader) -> RuntimeResult; - /// Get the file back from the buffered writer - fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult; -} - -pub trait FileInterfaceBufWrite: FileInterfaceWrite + FileInterfaceExt { - fn sync_write_cache(&mut self) -> RuntimeResult<()>; -} - -/// Readable object -pub trait FileInterfaceRead { - /// Read in a block of the exact given length - fn fread_exact_block(&mut self) -> RuntimeResult<[u8; N]> { - let mut ret = [0u8; N]; - self.fread_exact(&mut ret)?; - Ok(ret) - } - /// Read in `n` bytes to fill the given buffer - fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()>; -} - -/// Writable object -pub trait FileInterfaceWrite { - /// Attempt to write the buffer into this object, returning the number of bytes that were - /// written. It is **NOT GUARANTEED THAT ALL DATA WILL BE WRITTEN** - fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult; - /// Attempt to write the entire buffer into this object, returning the number of bytes written - /// - /// It is guaranteed that if the [`Result`] returned is [`Ok(())`], then the entire buffer was - /// written to disk. - fn fwrite_all_count(&mut self, buf: &[u8]) -> (u64, RuntimeResult<()>) { - let len = buf.len() as u64; - let mut written = 0; - while written != len { - match self.fwrite(buf) { - Ok(0) => { - return ( - written, - Err(IoError::new( - IoErrorKind::WriteZero, - format!("could only write {} of {} bytes", written, buf.len()), - ) - .into()), - ) - } - Ok(n) => written += n, - Err(e) => return (written, Err(e)), - } - } - (written, Ok(())) - } - /// Attempt to write the entire buffer into this object - /// - /// If this return [`Ok(())`] then it is guaranteed that all bytes have been written - fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()> { - self.fwrite_all_count(buf).1 - } -} - -/// Advanced write traits -pub trait FileInterfaceWriteExt { - /// Sync data and metadata for this file - fn fwext_sync_all(&mut self) -> RuntimeResult<()> { - Ok(()) - } - /// Sync data for this file - fn fwext_sync_data(&mut self) -> RuntimeResult<()> { - Ok(()) - } - /// Sync meta for this file - fn fwext_sync_meta(&mut self) -> RuntimeResult<()> { - Ok(()) - } - /// Truncate the size of the file to the given size - /// - /// - If `to` > actual file length: the file is zero padded to fill `to - len` - /// - If `to` < actual file length: the file is trimmed to the size `to` - fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()>; -} - -/// Advanced file access -pub trait FileInterfaceExt { - /// Returns the length of the file - fn fext_length(&self) -> RuntimeResult; - /// Returns the current cursor position of the file - fn fext_cursor(&mut self) -> RuntimeResult; - /// Seek by `from` bytes from the start of the file - fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()>; -} diff --git a/server/src/engine/storage/common/interface/mod.rs b/server/src/engine/storage/common/interface/mod.rs index 237a1d51..ea1247c9 100644 --- a/server/src/engine/storage/common/interface/mod.rs +++ b/server/src/engine/storage/common/interface/mod.rs @@ -30,7 +30,6 @@ //! traits that provide an unified API for all file systems irrespective of their base impl //! -pub mod fs_imp; +pub mod fs; #[cfg(test)] -pub mod fs_test; -pub mod fs_traits; +mod vfs; diff --git a/server/src/engine/storage/common/interface/fs_test.rs b/server/src/engine/storage/common/interface/vfs.rs similarity index 50% rename from server/src/engine/storage/common/interface/fs_test.rs rename to server/src/engine/storage/common/interface/vfs.rs index 00eecb79..d1f84f42 100644 --- a/server/src/engine/storage/common/interface/fs_test.rs +++ b/server/src/engine/storage/common/interface/vfs.rs @@ -1,5 +1,5 @@ /* - * Created on Sun Jan 07 2024 + * Created on Thu Feb 29 2024 * * This file is a part of Skytable * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source @@ -24,19 +24,10 @@ * */ -//! File system emulation -//! -//! This directory contains some implementations of virtual file systems (either emulating `/tmp` or -//! `/dev/null`) that are directly implemented at the application level with some necessary changes -//! required for testing -//! +#![allow(dead_code)] use { - super::fs_traits::{ - FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, FileInterfaceRead, - FileInterfaceWrite, FileInterfaceWriteExt, FileOpen, - }, - crate::engine::{sync::cell::Lazy, RuntimeResult}, + crate::{engine::sync::cell::Lazy, IoResult}, parking_lot::RwLock, std::{ collections::{ @@ -48,58 +39,130 @@ use { }; /* - vfs definitions + virtual fs impl + --- */ -/// # VirtualFS -/// -/// A virtual file system stored entirely in the process's memory (inclusive of swap if enabled; no explicit discrimination is made) -/// -/// The virtual file system is generally intended for being utilized as an in-memory filesystem, primarily for testing -/// purposes and has a lot of limitations. -/// -/// It has support for: -/// - nested directories -/// - read/write permissions -/// - file position tracking and seeking -/// - directory operations -pub struct VirtualFS; +/* + definitions + --- + fs, node, dir, file +*/ /// A virtual directory type VDir = HashMap, VNode>; /// An iterator over the components of a file path (alias) type ComponentIter<'a> = std::iter::Take>; -/** - vnode - --- - either a vfile or a vdir -*/ +pub struct VirtualFS { + root: HashMap, VNode>, +} + #[derive(Debug)] -pub(super) enum VNode { +enum VNode { Dir(HashMap, Self>), File(VFile), } -impl VNode { - fn as_dir_mut(&mut self) -> Option<&mut VDir> { - match self { - Self::Dir(d) => Some(d), - Self::File(_) => None, - } +#[derive(Debug)] +pub(super) struct VFile { + read: bool, + write: bool, + data: Vec, + pos: usize, +} + +#[derive(Debug, PartialEq)] +/// Result of opening a file +/// - Created: newly created file +/// - Existing: existing file that was reopened +pub enum FileOpen { + /// new file + Created(CF), + /// existing file + Existing(EF), +} + +pub struct VFileDescriptor(pub(super) Box); + +impl VFileDescriptor { + pub(super) fn get_ref<'a>(&self, vfs: &'a VirtualFS) -> &'a VFile { + vfs.with_file(&self.0, |f| Ok(f)).unwrap() + } + pub(super) fn get_mut<'a>(&self, vfs: &'a mut VirtualFS) -> &'a mut VFile { + vfs.with_file_mut(&self.0, |f| Ok(f)).unwrap() + } +} + +impl Drop for VFileDescriptor { + fn drop(&mut self) { + VirtualFS::instance() + .write() + .with_file_mut(&self.0, |f| { + f.pos = 0; + f.write = false; + f.read = false; + Ok(()) + }) + .unwrap() } } /* - vfile + impl */ -#[derive(Debug)] -pub struct VFile { - read: bool, - write: bool, - data: Vec, - pos: usize, +impl VFile { + pub fn truncate(&mut self, to: u64) -> IoResult<()> { + if !self.write { + return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into()); + } + if to as usize > self.data.len() { + self.data.resize(to as usize, 0); + } else { + self.data.truncate(to as usize); + } + if self.pos > self.data.len() { + self.pos = self.data.len(); + } + Ok(()) + } + pub fn length(&self) -> IoResult { + Ok(self.data.len() as u64) + } + pub fn cursor(&self) -> IoResult { + Ok(self.pos as u64) + } + pub fn seek_from_start(&mut self, by: u64) -> IoResult<()> { + if by > self.data.len() as u64 { + return Err(Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into()); + } + self.pos = by as usize; + Ok(()) + } + pub fn fread_exact(&mut self, buf: &mut [u8]) -> IoResult<()> { + if !self.read { + return Err(Error::new(ErrorKind::PermissionDenied, "Read permission denied").into()); + } + let available_bytes = self.current().len(); + if available_bytes < buf.len() { + return Err(Error::from(ErrorKind::UnexpectedEof).into()); + } + buf.copy_from_slice(&self.data[self.pos..self.pos + buf.len()]); + self.pos += buf.len(); + Ok(()) + } + pub fn fwrite(&mut self, bytes: &[u8]) -> IoResult { + if !self.write { + return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into()); + } + if self.pos + bytes.len() > self.data.len() { + self.data.resize(self.pos + bytes.len(), 0); + } + self.data[self.pos..self.pos + bytes.len()].copy_from_slice(bytes); + self.pos += bytes.len(); + Ok(bytes.len() as _) + } } impl VFile { @@ -114,171 +177,86 @@ impl VFile { fn current(&self) -> &[u8] { &self.data[self.pos..] } -} - -mod err { - //! Errors - //! - //! These are custom errors returned by the virtual file system - use { - crate::engine::RuntimeResult, - std::io::{Error, ErrorKind}, - }; - pub(super) fn item_is_not_file() -> RuntimeResult { - Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into()) - } - pub(super) fn file_in_dir_path() -> RuntimeResult { - Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into()) - } - pub(super) fn dir_missing_in_path() -> RuntimeResult { - Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into()) - } - pub(super) fn could_not_find_item() -> RuntimeResult { - Err(Error::new(ErrorKind::NotFound, "could not find item").into()) + fn fw_write_all(&mut self, bytes: &[u8]) -> IoResult<()> { + self.fwrite(bytes).map(|_| ()) } } -mod util { - use { - super::{err, ComponentIter, VDir, VNode}, - crate::engine::RuntimeResult, - }; - pub(super) fn split_parts(fpath: &str) -> Vec<&str> { - fpath.split("/").collect() - } - pub(super) fn split_target_and_components(fpath: &str) -> (&str, ComponentIter) { - let parts = split_parts(fpath); - let target = parts.last().unwrap(); - let component_len = parts.len() - 1; - (target, parts.into_iter().take(component_len)) - } - pub(super) fn find_target_dir_mut<'a>( - components: ComponentIter, - mut current: &'a mut VDir, - ) -> RuntimeResult<&'a mut VDir> { - for component in components { - match current.get_mut(component) { - Some(VNode::Dir(d)) => current = d, - Some(VNode::File(_)) => return err::file_in_dir_path(), - None => return err::dir_missing_in_path(), - } - } - Ok(current) - } - pub(super) fn find_target_dir<'a>( - components: ComponentIter, - mut current: &'a VDir, - ) -> RuntimeResult<&'a VDir> { - for component in components { - match current.get(component) { - Some(VNode::Dir(d)) => current = d, - Some(VNode::File(_)) => return err::file_in_dir_path(), - None => return err::dir_missing_in_path(), - } +impl VNode { + fn as_dir_mut(&mut self) -> Option<&mut VDir> { + match self { + Self::Dir(d) => Some(d), + Self::File(_) => None, } - Ok(current) } } -/* - vfs impl: - - nested directory structure - - make parents - - make child -*/ - impl VirtualFS { - pub fn fetch_raw_data(file_name: &str) -> RuntimeResult> { - Self::with_file(file_name, |f| Ok(f.data.clone())) + pub fn instance() -> &'static RwLock { + static GLOBAL_VFS: Lazy, fn() -> RwLock> = + Lazy::new(|| RwLock::new(VirtualFS::new())); + &GLOBAL_VFS } - /// Get a handle to the virtual filesystem - fn handle() -> &'static RwLock { - static VFS: Lazy, fn() -> RwLock> = Lazy::new(|| Default::default()); - &VFS + pub fn get_data(&self, path: &str) -> IoResult> { + self.with_file(path, |f| Ok(f.data.clone())) } - fn with_file_mut( - fpath: &str, - mut f: impl FnMut(&mut VFile) -> RuntimeResult, - ) -> RuntimeResult { - let mut vfs = Self::handle().write(); + pub fn fs_fcreate_rw(&mut self, fpath: &str) -> IoResult { let (target_file, components) = util::split_target_and_components(fpath); - let target_dir = util::find_target_dir_mut(components, &mut vfs)?; - match target_dir.get_mut(target_file) { - Some(VNode::File(file)) => f(file), - Some(VNode::Dir(_)) => return err::item_is_not_file(), - None => return Err(Error::from(ErrorKind::NotFound).into()), - } - } - fn with_file( - fpath: &str, - mut f: impl FnMut(&VFile) -> RuntimeResult, - ) -> RuntimeResult { - let vfs = Self::handle().read(); - let (target_file, components) = util::split_target_and_components(fpath); - let target_dir = util::find_target_dir(components, &vfs)?; - match target_dir.get(target_file) { - Some(VNode::File(file)) => f(file), - Some(VNode::Dir(_)) => return err::item_is_not_file(), - None => return Err(Error::from(ErrorKind::NotFound).into()), - } - } - fn with_item_mut( - fpath: &str, - f: impl Fn(OccupiedEntry, VNode>) -> RuntimeResult, - ) -> RuntimeResult { - let mut vfs = Self::handle().write(); - let mut current = &mut *vfs; - // process components - let (target, components) = util::split_target_and_components(fpath); - for component in components { - match current.get_mut(component) { - Some(VNode::Dir(dir)) => { - current = dir; + let target_dir = util::find_target_dir_mut(components, &mut self.root)?; + match target_dir.entry(target_file.into()) { + Entry::Occupied(k) => { + match k.get() { + VNode::Dir(_) => { + return Err(Error::new( + ErrorKind::AlreadyExists, + "found directory with same name where file was to be created", + ) + .into()); + } + VNode::File(_) => { + // the file already exists + return Err(Error::new( + ErrorKind::AlreadyExists, + "the file already exists", + ) + .into()); + } } - Some(VNode::File(_)) => return err::file_in_dir_path(), - None => return err::dir_missing_in_path(), } - } - match current.entry(target.into()) { - Entry::Occupied(item) => return f(item), - Entry::Vacant(_) => return err::could_not_find_item(), + Entry::Vacant(v) => { + // no file exists, we can create this + v.insert(VNode::File(VFile::new(true, true, vec![], 0))); + Ok(VFileDescriptor(fpath.into())) + } } } - fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> RuntimeResult<()> { - Self::with_item_mut(fpath, |node| match node.get() { - VNode::Dir(d) => { - if allow_if_non_empty || d.is_empty() { - node.remove(); - return Ok(()); - } - return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into()); - } - VNode::File(_) => return err::file_in_dir_path(), + pub fn fs_fopen_rw(&mut self, fpath: &str) -> IoResult { + self.with_file_mut(fpath, |f| { + f.read = true; + f.write = true; + Ok(VFileDescriptor(fpath.into())) }) } -} - -impl FSInterface for VirtualFS { - type File = VFileDescriptor; - fn fs_rename(from: &str, to: &str) -> RuntimeResult<()> { + pub fn fs_rename(&mut self, from: &str, to: &str) -> IoResult<()> { // get file data - let data = VirtualFS::with_file(from, |f| Ok(f.data.clone()))?; + let data = self.with_file(from, |f| Ok(f.data.clone()))?; // create new file - let file = VirtualFS::fs_fopen_or_create_rw(to)?; + let file = self.fs_fopen_or_create_rw(to)?; match file { - FileOpen::Created(mut c) => { - c.fw_write_all(&data)?; + FileOpen::Created(c) => { + c.get_mut(self).fw_write_all(&data)?; } - FileOpen::Existing(mut e) => { - e.fwext_truncate_to(0)?; - e.fw_write_all(&data)?; + FileOpen::Existing(c) => { + let file = c.get_mut(self); + file.truncate(0)?; + file.fw_write_all(&data)?; } } // delete old file - Self::fs_remove_file(from) + self.fs_remove_file(from) } - fn fs_remove_file(fpath: &str) -> RuntimeResult<()> { - VirtualFS::with_item_mut(fpath, |e| match e.get() { + pub fn fs_remove_file(&mut self, fpath: &str) -> IoResult<()> { + self.with_item_mut(fpath, |e| match e.get() { VNode::File(_) => { e.remove(); Ok(()) @@ -286,11 +264,9 @@ impl FSInterface for VirtualFS { _ => return err::item_is_not_file(), }) } - fn fs_create_dir(fpath: &str) -> RuntimeResult<()> { - // get vfs - let mut vfs = VirtualFS::handle().write(); + pub fn fs_create_dir(&mut self, fpath: &str) -> IoResult<()> { // get root dir - let mut current = &mut *vfs; + let mut current = &mut self.root; // process components let (target, mut components) = util::split_target_and_components(fpath); while let Some(component) = components.next() { @@ -310,9 +286,8 @@ impl FSInterface for VirtualFS { } } } - fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> { - let mut vfs = VirtualFS::handle().write(); - fn create_ahead(mut ahead: &[&str], current: &mut VDir) -> RuntimeResult<()> { + pub fn fs_create_dir_all(&mut self, fpath: &str) -> IoResult<()> { + fn create_ahead(mut ahead: &[&str], current: &mut VDir) -> IoResult<()> { if ahead.is_empty() { return Ok(()); } @@ -335,19 +310,26 @@ impl FSInterface for VirtualFS { } } let pieces = util::split_parts(fpath); - create_ahead(&pieces, &mut *vfs) + create_ahead(&pieces, &mut self.root) } - fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> { - VirtualFS::delete_dir(fpath, false) + pub fn fs_delete_dir(&mut self, fpath: &str) -> IoResult<()> { + self.dir_delete(fpath, false) } - fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> { - VirtualFS::delete_dir(fpath, true) + pub fn fs_delete_dir_all(&mut self, fpath: &str) -> IoResult<()> { + self.dir_delete(fpath, true) + } +} + +impl VirtualFS { + fn new() -> Self { + Self { + root: HashMap::new(), + } } - fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult> { - let mut vfs = VirtualFS::handle().write(); + fn fs_fopen_or_create_rw(&mut self, fpath: &str) -> IoResult> { // components let (target_file, components) = util::split_target_and_components(fpath); - let target_dir = util::find_target_dir_mut(components, &mut vfs)?; + let target_dir = util::find_target_dir_mut(components, &mut self.root)?; match target_dir.entry(target_file.into()) { Entry::Occupied(mut oe) => match oe.get_mut() { VNode::File(f) => { @@ -363,248 +345,128 @@ impl FSInterface for VirtualFS { } } } - fn fs_fcreate_rw(fpath: &str) -> RuntimeResult { - let mut vfs = VirtualFS::handle().write(); + fn with_file_mut<'a, T>( + &'a mut self, + fpath: &str, + mut f: impl FnMut(&'a mut VFile) -> IoResult, + ) -> IoResult { let (target_file, components) = util::split_target_and_components(fpath); - let target_dir = util::find_target_dir_mut(components, &mut vfs)?; - match target_dir.entry(target_file.into()) { - Entry::Occupied(k) => { - match k.get() { - VNode::Dir(_) => { - return Err(Error::new( - ErrorKind::AlreadyExists, - "found directory with same name where file was to be created", - ) - .into()); - } - VNode::File(_) => { - // the file already exists - return Err(Error::new( - ErrorKind::AlreadyExists, - "the file already exists", - ) - .into()); - } - } - } - Entry::Vacant(v) => { - // no file exists, we can create this - v.insert(VNode::File(VFile::new(true, true, vec![], 0))); - Ok(VFileDescriptor(fpath.into())) - } + let target_dir = util::find_target_dir_mut(components, &mut self.root)?; + match target_dir.get_mut(target_file) { + Some(VNode::File(file)) => f(file), + Some(VNode::Dir(_)) => return err::item_is_not_file(), + None => return Err(Error::from(ErrorKind::NotFound).into()), } } - fn fs_fopen_rw(fpath: &str) -> RuntimeResult { - VirtualFS::with_file_mut(fpath, |f| { - f.read = true; - f.write = true; - Ok(VFileDescriptor(fpath.into())) - }) - } -} - -/* - vfile & descriptor impls - (this is our `File` but a temporary, completely in-memory file) -*/ - -pub struct VFileDescriptor(Box); -impl Drop for VFileDescriptor { - fn drop(&mut self) { - let _ = VirtualFS::with_file_mut(&self.0, |f| { - f.read = false; - f.write = false; - f.pos = 0; - Ok(()) - }); - } -} - -impl FileInterface for VFileDescriptor { - type BufReader = Self; - type BufWriter = Self; - fn upgrade_to_buffered_reader(self) -> RuntimeResult { - Ok(self) - } - fn upgrade_to_buffered_writer(self) -> RuntimeResult { - Ok(self) - } - fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { - Ok(r) - } - fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult { - Ok(r) - } -} - -impl FileInterfaceRead for VFileDescriptor { - fn fread_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { - VirtualFS::with_file_mut(&self.0, |file| { - if !file.read { - return Err( - Error::new(ErrorKind::PermissionDenied, "Read permission denied").into(), - ); - } - let available_bytes = file.current().len(); - if available_bytes < buf.len() { - return Err(Error::from(ErrorKind::UnexpectedEof).into()); - } - buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]); - file.pos += buf.len(); - Ok(()) - }) - } -} - -impl FileInterfaceWrite for VFileDescriptor { - fn fwrite(&mut self, bytes: &[u8]) -> RuntimeResult { - VirtualFS::with_file_mut(&self.0, |file| { - if !file.write { - return Err( - Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(), - ); - } - if file.pos + bytes.len() > file.data.len() { - file.data.resize(file.pos + bytes.len(), 0); - } - file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes); - file.pos += bytes.len(); - Ok(bytes.len() as _) - }) + fn with_file<'a, T>( + &'a self, + fpath: &str, + mut f: impl FnMut(&'a VFile) -> IoResult, + ) -> IoResult { + let (target_file, components) = util::split_target_and_components(fpath); + let target_dir = util::find_target_dir(components, &self.root)?; + match target_dir.get(target_file) { + Some(VNode::File(file)) => f(file), + Some(VNode::Dir(_)) => return err::item_is_not_file(), + None => return Err(Error::from(ErrorKind::NotFound).into()), + } } -} - -impl FileInterfaceWriteExt for VFileDescriptor { - fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> { - VirtualFS::with_file_mut(&self.0, |file| { - if !file.write { - return Err( - Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(), - ); - } - if to as usize > file.data.len() { - file.data.resize(to as usize, 0); - } else { - file.data.truncate(to as usize); - } - if file.pos > file.data.len() { - file.pos = file.data.len(); + fn with_item_mut( + &mut self, + fpath: &str, + f: impl Fn(OccupiedEntry, VNode>) -> IoResult, + ) -> IoResult { + // process components + let (target, components) = util::split_target_and_components(fpath); + let mut current = &mut self.root; + for component in components { + match current.get_mut(component) { + Some(VNode::Dir(dir)) => { + current = dir; + } + Some(VNode::File(_)) => return err::file_in_dir_path(), + None => return err::dir_missing_in_path(), } - Ok(()) - }) - } -} - -impl FileInterfaceBufWrite for VFileDescriptor { - fn sync_write_cache(&mut self) -> RuntimeResult<()> { - Ok(()) + } + match current.entry(target.into()) { + Entry::Occupied(item) => return f(item), + Entry::Vacant(_) => return err::could_not_find_item(), + } } -} - -impl FileInterfaceExt for VFileDescriptor { - fn fext_length(&self) -> RuntimeResult { - VirtualFS::with_file(&self.0, |f| Ok(f.data.len() as u64)) - } - fn fext_cursor(&mut self) -> RuntimeResult { - VirtualFS::with_file(&self.0, |f| Ok(f.pos as u64)) - } - fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> { - VirtualFS::with_file_mut(&self.0, |file| { - if by > file.data.len() as u64 { - return Err( - Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into(), - ); + fn dir_delete(&mut self, fpath: &str, allow_if_non_empty: bool) -> IoResult<()> { + self.with_item_mut(fpath, |node| match node.get() { + VNode::Dir(d) => { + if allow_if_non_empty || d.is_empty() { + node.remove(); + return Ok(()); + } + return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into()); } - file.pos = by as usize; - Ok(()) + VNode::File(_) => return err::file_in_dir_path(), }) } } -/// An application level implementation of `/dev/null` with some changes -pub struct NullFS; -/// A handle to a file in `/dev/null` (emulated) -pub struct NullFile; - -impl FSInterface for NullFS { - type File = NullFile; - fn fs_remove_file(_: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_rename(_: &str, _: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_create_dir(_: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_create_dir_all(_: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_delete_dir(_: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_delete_dir_all(_: &str) -> RuntimeResult<()> { - Ok(()) - } - fn fs_fopen_or_create_rw(_: &str) -> RuntimeResult> { - Ok(FileOpen::Created(NullFile)) - } - fn fs_fopen_rw(_: &str) -> RuntimeResult { - Ok(NullFile) - } - fn fs_fcreate_rw(_: &str) -> RuntimeResult { - Ok(NullFile) - } -} - -impl FileInterface for NullFile { - type BufReader = Self; - type BufWriter = Self; - fn upgrade_to_buffered_reader(self) -> RuntimeResult { - Ok(self) - } - fn upgrade_to_buffered_writer(self) -> RuntimeResult { - Ok(self) - } - fn downgrade_reader(r: Self::BufReader) -> RuntimeResult { - Ok(r) - } - fn downgrade_writer(r: Self::BufWriter) -> RuntimeResult { - Ok(r) +mod err { + //! Errors + //! + //! These are custom errors returned by the virtual file system + use { + crate::IoResult, + std::io::{Error, ErrorKind}, + }; + pub(super) fn item_is_not_file() -> IoResult { + Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into()) } -} - -impl FileInterfaceWrite for NullFile { - fn fwrite(&mut self, buf: &[u8]) -> RuntimeResult { - Ok(buf.len() as _) + pub(super) fn file_in_dir_path() -> IoResult { + Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into()) } -} - -impl FileInterfaceWriteExt for NullFile { - fn fwext_truncate_to(&mut self, _: u64) -> RuntimeResult<()> { - Ok(()) + pub(super) fn dir_missing_in_path() -> IoResult { + Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into()) } -} - -impl FileInterfaceRead for NullFile { - fn fread_exact(&mut self, _: &mut [u8]) -> RuntimeResult<()> { - Ok(()) + pub(super) fn could_not_find_item() -> IoResult { + Err(Error::new(ErrorKind::NotFound, "could not find item").into()) } } -impl FileInterfaceExt for NullFile { - fn fext_length(&self) -> RuntimeResult { - Ok(0) +mod util { + use { + super::{err, ComponentIter, VDir, VNode}, + crate::IoResult, + }; + pub(super) fn split_parts(fpath: &str) -> Vec<&str> { + fpath.split("/").collect() } - fn fext_cursor(&mut self) -> RuntimeResult { - Ok(0) + pub(super) fn split_target_and_components(fpath: &str) -> (&str, ComponentIter) { + let parts = split_parts(fpath); + let target = parts.last().unwrap(); + let component_len = parts.len() - 1; + (target, parts.into_iter().take(component_len)) } - fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> RuntimeResult<()> { - Ok(()) + pub(super) fn find_target_dir_mut<'a>( + components: ComponentIter, + mut current: &'a mut VDir, + ) -> IoResult<&'a mut VDir> { + for component in components { + match current.get_mut(component) { + Some(VNode::Dir(d)) => current = d, + Some(VNode::File(_)) => return err::file_in_dir_path(), + None => return err::dir_missing_in_path(), + } + } + Ok(current) } -} -impl FileInterfaceBufWrite for NullFile { - fn sync_write_cache(&mut self) -> RuntimeResult<()> { - Ok(()) + pub(super) fn find_target_dir<'a>( + components: ComponentIter, + mut current: &'a VDir, + ) -> IoResult<&'a VDir> { + for component in components { + match current.get(component) { + Some(VNode::Dir(d)) => current = d, + Some(VNode::File(_)) => return err::file_in_dir_path(), + None => return err::dir_missing_in_path(), + } + } + Ok(current) } } diff --git a/server/src/engine/storage/common/sdss/impls/sdss_r1/mod.rs b/server/src/engine/storage/common/sdss/impls/sdss_r1/mod.rs index 49f9e423..b4de9a09 100644 --- a/server/src/engine/storage/common/sdss/impls/sdss_r1/mod.rs +++ b/server/src/engine/storage/common/sdss/impls/sdss_r1/mod.rs @@ -34,13 +34,18 @@ pub mod rw; use { super::super::super::{ - interface::fs_traits::{FileInterfaceRead, FileInterfaceWrite}, static_meta::{HostArch, HostEndian, HostOS, HostPointerWidth, SDSS_MAGIC_8B}, versions::{self, DriverVersion, FileSpecifierVersion, HeaderVersion, ServerVersion}, }, crate::{ - engine::{error::StorageError, mem::unsafe_apis::memcpy, RuntimeResult}, + engine::{ + error::StorageError, + mem::unsafe_apis::memcpy, + storage::common::interface::fs::{FileRead, FileWrite}, + RuntimeResult, + }, util::os, + IoResult, }, std::{ mem::{transmute, ManuallyDrop}, @@ -412,17 +417,14 @@ pub trait FileSpecV1 { ) -> RuntimeResult; /// read and validate metadata (only override if you need to) fn read_metadata( - f: &mut impl FileInterfaceRead, + f: &mut impl FileRead, v_args: Self::DecodeArgs, ) -> RuntimeResult { let md = HeaderV1::decode(f.fread_exact_block()?)?; Self::validate_metadata(md, v_args) } /// write metadata - fn write_metadata( - f: &mut impl FileInterfaceWrite, - args: Self::EncodeArgs, - ) -> RuntimeResult; + fn write_metadata(f: &mut impl FileWrite, args: Self::EncodeArgs) -> IoResult; fn metadata_to_block(args: Self::EncodeArgs) -> RuntimeResult> { let mut v = Vec::new(); Self::write_metadata(&mut v, args)?; @@ -479,15 +481,12 @@ impl FileSpecV1 for Sfs { Err(StorageError::HeaderDecodeVersionMismatch.into()) } } - fn write_metadata( - f: &mut impl FileInterfaceWrite, - _: Self::EncodeArgs, - ) -> RuntimeResult { + fn write_metadata(f: &mut impl FileWrite, _: Self::EncodeArgs) -> IoResult { let (md, block) = HeaderV1::::encode_return( Self::FILE_CLASS, Self::FILE_SPECIFIER, Self::FILE_SPECFIER_VERSION, ); - f.fw_write_all(&block).map(|_| md) + f.fwrite_all(&block).map(|_| md) } } diff --git a/server/src/engine/storage/common/sdss/impls/sdss_r1/rw.rs b/server/src/engine/storage/common/sdss/impls/sdss_r1/rw.rs index 348bb384..72bec469 100644 --- a/server/src/engine/storage/common/sdss/impls/sdss_r1/rw.rs +++ b/server/src/engine/storage/common/sdss/impls/sdss_r1/rw.rs @@ -32,15 +32,13 @@ use { mem::fixed_vec::FixedVec, storage::common::{ checksum::SCrc64, - interface::fs_traits::{ - FSInterface, FileInterface, FileInterfaceBufWrite, FileInterfaceExt, - FileInterfaceRead, FileInterfaceWrite, FileInterfaceWriteExt, FileOpen, - }, + interface::fs::{BufferedReader, File, FileExt, FileRead, FileWrite, FileWriteExt}, sdss::sdss_r1::FileSpecV1, }, RuntimeResult, }, util::os::SysIOError, + IoResult, }, std::mem, }; @@ -51,109 +49,56 @@ use { #[derive(Debug, PartialEq)] /// A file with it's layout defined by a SDSS file specification -pub struct SdssFile { +pub struct SdssFile { file: F, meta: S::Metadata, } -impl SdssFile { +impl SdssFile { fn new(file: F, meta: S::Metadata) -> Self { Self { file, meta } } - /// Returns the SDSS metadata associated with this file - pub fn sdss_metadata(&self) -> &S::Metadata { - &self.meta - } } -impl SdssFile { +impl SdssFile { /// Open an existing SDSS based file (with no validation arguments) - pub fn open>(path: &str) -> RuntimeResult + pub fn open(path: &str) -> RuntimeResult where S: FileSpecV1, { - let mut f = Fs::fs_fopen_rw(path)?; + let mut f = File::open(path)?; let md = S::read_metadata(&mut f, ())?; Ok(Self::new(f, md)) } /// Create a new SDSS based file (with no initialization arguments) - pub fn create>(path: &str) -> RuntimeResult + pub fn create(path: &str) -> RuntimeResult where S: FileSpecV1, { - let mut f = Fs::fs_fcreate_rw(path)?; + let mut f = File::create(path)?; let md = S::write_metadata(&mut f, ())?; Ok(Self::new(f, md)) } - /// Create or open an SDSS based file (with no initialization or validation arguments) - pub fn open_or_create_perm_rw>( - path: &str, - ) -> RuntimeResult> - where - S: FileSpecV1, - { - match Fs::fs_fopen_or_create_rw(path)? { - FileOpen::Created(mut new) => { - let md = S::write_metadata(&mut new, ())?; - Ok(FileOpen::Created(Self::new(new, md))) - } - FileOpen::Existing(mut existing) => { - let md = S::read_metadata(&mut existing, ())?; - Ok(FileOpen::Existing(Self::new(existing, md))) - } - } - } -} - -impl SdssFile { - /// Get a buffered reader. Use [`SdssFile::downgrade_reader`] to get back the original file - pub fn into_buffered_reader(self) -> RuntimeResult> { + pub fn into_buffered_reader(self) -> IoResult> { let Self { file, meta } = self; - let bufreader = F::upgrade_to_buffered_reader(file)?; - Ok(SdssFile::new(bufreader, meta)) - } - /// Get back the original file from the buffered reader - pub fn downgrade_reader( - SdssFile { file, meta }: SdssFile, - ) -> RuntimeResult { - let me = F::downgrade_reader(file)?; - Ok(Self::new(me, meta)) - } - /// Get a buffered writer. Use [`SdssFile::downgrade_writer`] to get back the original file - pub fn into_buffered_writer(self) -> RuntimeResult> { - let Self { file, meta } = self; - let bufwriter = F::upgrade_to_buffered_writer(file)?; - Ok(SdssFile::new(bufwriter, meta)) + let r = file.into_buffered_reader(); + Ok(SdssFile::new(r, meta)) } - /// Get back the original file from the buffered writer - /// - /// NB: THis will usually not explicitly sync any pending data, unless the downgrade implementation - /// of the interface does so - pub fn downgrade_writer( - SdssFile { file, meta }: SdssFile, - ) -> RuntimeResult { - let me = F::downgrade_writer(file)?; - Ok(Self::new(me, meta)) - } -} - -impl SdssFile { - /// Sync writes - pub fn sync_writes(&mut self) -> RuntimeResult<()> { - self.file.sync_write_cache() + pub fn downgrade_reader(SdssFile { file, meta }: SdssFile) -> Self { + Self::new(file.into_inner(), meta) } } -impl SdssFile { +impl SdssFile { /// Attempt to fill the entire buffer from the file - pub fn read_buffer(&mut self, buffer: &mut [u8]) -> RuntimeResult<()> { + pub fn read_buffer(&mut self, buffer: &mut [u8]) -> IoResult<()> { self.file.fread_exact(buffer) } } -impl SdssFile { +impl SdssFile { /// Read the entire part of the remaining file into memory - pub fn read_full(&mut self) -> RuntimeResult> { + pub fn read_full(&mut self) -> IoResult> { let len = self.file_length()? - self.file_cursor()?; let mut buf = vec![0; len as usize]; self.read_buffer(&mut buf)?; @@ -161,38 +106,38 @@ impl SdssFile { } } -impl SdssFile { +impl SdssFile { /// Get the current position of the file - pub fn file_cursor(&mut self) -> RuntimeResult { - self.file.fext_cursor() + pub fn file_cursor(&mut self) -> IoResult { + self.file.f_cursor() } /// Get the length of the file - pub fn file_length(&self) -> RuntimeResult { - self.file.fext_length() + pub fn file_length(&self) -> IoResult { + self.file.f_len() } /// Move the cursor `n` bytes from the start - pub fn seek_from_start(&mut self, n: u64) -> RuntimeResult<()> { - self.file.fext_seek_ahead_from_start_by(n) + pub fn seek_from_start(&mut self, n: u64) -> IoResult<()> { + self.file.f_seek_start(n) } } -impl SdssFile { +impl SdssFile { /// Attempt to write the entire buffer into the file - pub fn write_buffer(&mut self, data: &[u8]) -> RuntimeResult<()> { - self.file.fw_write_all(data) + pub fn write_buffer(&mut self, data: &[u8]) -> IoResult<()> { + self.file.fwrite_all(data) } } -impl SdssFile { +impl SdssFile { /// Sync all data and metadata permanently - pub fn fsync_all(&mut self) -> RuntimeResult<()> { - self.file.fwext_sync_all()?; + pub fn fsync_all(&mut self) -> IoResult<()> { + self.file.fsync_all()?; Ok(()) } /// Write a block followed by an explicit fsync call - pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> { - self.file.fw_write_all(data)?; - self.file.fwext_sync_all() + pub fn fsynced_write(&mut self, data: &[u8]) -> IoResult<()> { + self.file.fwrite_all(data)?; + self.file.fsync_all() } } @@ -202,32 +147,32 @@ impl SdssFile { - f: SdssFile, +pub struct TrackedReader { + f: SdssFile, len: u64, cursor: u64, cs: SCrc64, } -pub struct TrackedReaderContext<'a, F, S: FileSpecV1> { - tr: &'a mut TrackedReader, +pub struct TrackedReaderContext<'a, S: FileSpecV1> { + tr: &'a mut TrackedReader, p_checksum: SCrc64, } -impl<'a, F: FileInterfaceRead, S: FileSpecV1> TrackedReaderContext<'a, F, S> { - pub fn read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { +impl<'a, S: FileSpecV1> TrackedReaderContext<'a, S> { + pub fn read(&mut self, buf: &mut [u8]) -> IoResult<()> { self.tr .tracked_read(buf) .map(|_| self.p_checksum.update(buf)) } - pub fn read_block(&mut self) -> RuntimeResult<[u8; N]> { + pub fn read_block(&mut self) -> IoResult<[u8; N]> { let mut block = [0; N]; self.tr.tracked_read(&mut block).map(|_| { self.p_checksum.update(&block); block }) } - pub fn finish(self) -> (u64, &'a mut TrackedReader) { + pub fn finish(self) -> (u64, &'a mut TrackedReader) { let Self { tr, p_checksum } = self; (p_checksum.finish(), tr) } @@ -236,15 +181,12 @@ impl<'a, F: FileInterfaceRead, S: FileSpecV1> TrackedReaderContext<'a, F, S> { } } -impl TrackedReader { +impl TrackedReader { /// Create a new [`TrackedReader`]. This needs to retrieve file position and length - pub fn new(mut f: SdssFile) -> RuntimeResult> { + pub fn new(mut f: SdssFile) -> IoResult> { f.file_cursor().and_then(|c| Self::with_cursor(f, c)) } - pub fn with_cursor( - f: SdssFile, - cursor: u64, - ) -> RuntimeResult> { + pub fn with_cursor(f: SdssFile, cursor: u64) -> IoResult { let len = f.file_length()?; let f = f.into_buffered_reader()?; Ok(TrackedReader { @@ -256,19 +198,19 @@ impl TrackedReader { } } -impl TrackedReader { - pub fn context(&mut self) -> TrackedReaderContext { +impl TrackedReader { + pub fn context(&mut self) -> TrackedReaderContext { TrackedReaderContext { tr: self, p_checksum: SCrc64::new(), } } /// Attempt to fill the buffer. This read is tracked. - pub fn tracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + pub fn tracked_read(&mut self, buf: &mut [u8]) -> IoResult<()> { self.untracked_read(buf).map(|_| self.cs.update(buf)) } /// Attempt to read a byte. This read is also tracked. - pub fn read_byte(&mut self) -> RuntimeResult { + pub fn read_byte(&mut self) -> IoResult { let mut buf = [0u8; 1]; self.tracked_read(&mut buf).map(|_| buf[0]) } @@ -281,7 +223,7 @@ impl TrackedReader { /// Do an untracked read of the file. /// /// NB: The change in cursor however will still be tracked. - pub fn untracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + pub fn untracked_read(&mut self, buf: &mut [u8]) -> IoResult<()> { if self.remaining() >= buf.len() as u64 { match self.f.read_buffer(buf) { Ok(()) => { @@ -291,20 +233,20 @@ impl TrackedReader { Err(e) => return Err(e), } } else { - Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()) + Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into_inner()) } } /// Tracked read of a given block size. Shorthand for [`Self::tracked_read`] - pub fn read_block(&mut self) -> RuntimeResult<[u8; N]> { + pub fn read_block(&mut self) -> IoResult<[u8; N]> { if !self.has_left(N as _) { - return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()); + return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into_inner()); } let mut buf = [0; N]; self.tracked_read(&mut buf)?; Ok(buf) } /// Tracked read of a [`u64`] value - pub fn read_u64_le(&mut self) -> RuntimeResult { + pub fn read_u64_le(&mut self) -> IoResult { Ok(u64::from_le_bytes(self.read_block()?)) } pub fn current_checksum(&self) -> u64 { @@ -318,9 +260,9 @@ impl TrackedReader { } } -impl TrackedReader { +impl TrackedReader { /// Returns the base [`SdssFile`] - pub fn into_inner>(self) -> RuntimeResult> { + pub fn into_inner(self) -> SdssFile { SdssFile::downgrade_reader(self.f) } /// Returns the number of remaining bytes @@ -407,7 +349,7 @@ impl< } impl< - F: FileInterfaceExt, + F: FileExt, S: FileSpecV1, const SIZE: usize, const PANIC_IF_UNFLUSHED: bool, @@ -418,18 +360,18 @@ impl< /// /// NB: The cursor is fetched. If the cursor is already available, use [`Self::with_cursor`] pub fn new( - mut f: SdssFile, - ) -> RuntimeResult> + mut f: SdssFile, + ) -> IoResult> { f.file_cursor().map(|v| TrackedWriter::with_cursor(f, v)) } /// Create a new tracked writer with the provided cursor - pub fn with_cursor(f: SdssFile, c: u64) -> Self { + pub fn with_cursor(f: SdssFile, c: u64) -> Self { Self::with_cursor_and_checksum(f, c, SCrc64::new()) } /// Create a new tracked writer with the provided checksum and cursor pub fn with_cursor_and_checksum( - SdssFile { file, meta }: SdssFile, + SdssFile { file, meta }: SdssFile, c: u64, ck: SCrc64, ) -> Self { @@ -441,7 +383,7 @@ impl< } impl< - F: FileInterfaceWrite, + F: FileWrite, S: FileSpecV1, const SIZE: usize, const PANIC_IF_UNFLUSHED: bool, @@ -449,7 +391,7 @@ impl< > TrackedWriter { /// Same as [`Self::tracked_write_through_buffer`], but the partial state is updated - pub fn dtrack_write_through_buffer(&mut self, buf: &[u8]) -> RuntimeResult<()> { + pub fn dtrack_write_through_buffer(&mut self, buf: &[u8]) -> IoResult<()> { self.tracked_write_through_buffer(buf) .map(|_| self.t_partial_checksum.update(buf)) } @@ -458,7 +400,7 @@ impl< /// NB: /// - If errored, the number of bytes written are still tracked /// - If errored, the checksum is updated to reflect the number of bytes written (unless otherwise configured) - pub fn tracked_write_through_buffer(&mut self, buf: &[u8]) -> RuntimeResult<()> { + pub fn tracked_write_through_buffer(&mut self, buf: &[u8]) -> IoResult<()> { debug_assert!(self.buf.is_empty()); match self.f_d.fwrite_all_count(buf) { (cnt, r) => { @@ -475,7 +417,7 @@ impl< } } /// Same as [`Self::tracked_write`], but the partial state is updated - pub fn dtrack_write(&mut self, buf: &[u8]) -> RuntimeResult<()> { + pub fn dtrack_write(&mut self, buf: &[u8]) -> IoResult<()> { self.tracked_write(buf) .map(|_| self.t_partial_checksum.update(buf)) } @@ -487,7 +429,7 @@ impl< /// /// On error, if block error checksumming is set then whatever part of the block was written /// will be updated in the checksum. If disabled, then the checksum is unchanged. - pub fn tracked_write(&mut self, buf: &[u8]) -> RuntimeResult<()> { + pub fn tracked_write(&mut self, buf: &[u8]) -> IoResult<()> { let cursor_start = self.cursor_usize(); match self.untracked_write(buf) { Ok(()) => { @@ -504,7 +446,7 @@ impl< } } /// Do an untracked write - pub fn untracked_write(&mut self, buf: &[u8]) -> RuntimeResult<()> { + pub fn untracked_write(&mut self, buf: &[u8]) -> IoResult<()> { if self.available_capacity() >= buf.len() { unsafe { // UNSAFE(@ohsayan): above branch guarantees that we have sufficient space @@ -529,14 +471,14 @@ impl< Ok(()) } /// Flush the buffer and then sync data and metadata - pub fn flush_sync(&mut self) -> RuntimeResult<()> + pub fn flush_sync(&mut self) -> IoResult<()> where - F: FileInterfaceWriteExt, + F: FileWriteExt, { self.flush_buf().and_then(|_| self.fsync()) } /// Flush the buffer - pub fn flush_buf(&mut self) -> RuntimeResult<()> { + pub fn flush_buf(&mut self) -> IoResult<()> { match self.f_d.fwrite_all_count(&self.buf) { (written, r) => { if written as usize == self.buf.len() { @@ -560,11 +502,11 @@ impl< } } } - pub fn fsync(&mut self) -> RuntimeResult<()> + pub fn fsync(&mut self) -> IoResult<()> where - F: FileInterfaceWriteExt, + F: FileWriteExt, { - self.f_d.fwext_sync_all() + self.f_d.fsync_all() } } @@ -586,11 +528,11 @@ impl< #[test] fn check_vfs_buffering() { use crate::engine::storage::{ - common::interface::fs_test::{VFileDescriptor, VirtualFS}, + common::interface::fs::FileSystem, v2::raw::spec::{Header, SystemDatabaseV1}, }; fn rawfile() -> Vec { - VirtualFS::fetch_raw_data("myfile").unwrap() + FileSystem::read("myfile").unwrap() } let compiled_header = SystemDatabaseV1::metadata_to_block(()).unwrap(); let expected_checksum = { @@ -602,8 +544,8 @@ fn check_vfs_buffering() { }; closure! { // init writer - let mut twriter: TrackedWriter = - TrackedWriter::new(SdssFile::create::("myfile")?)?; + let mut twriter: TrackedWriter = + TrackedWriter::new(SdssFile::create("myfile")?)?; assert_eq!(twriter.cursor_usize(), Header::SIZE); { // W8192: write exact bufsize block; nothing is written (except SDSS header) diff --git a/server/src/engine/storage/common_encoding/r1/impls/gns/tests/full_chain.rs b/server/src/engine/storage/common_encoding/r1/impls/gns/tests/full_chain.rs index 41d57204..589c7474 100644 --- a/server/src/engine/storage/common_encoding/r1/impls/gns/tests/full_chain.rs +++ b/server/src/engine/storage/common_encoding/r1/impls/gns/tests/full_chain.rs @@ -71,11 +71,11 @@ fn create_space() { let uuid; // start 1 { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); uuid = init_space(&global, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); let spaces = global.state().idx().read(); let space = spaces.get("myspace").unwrap(); assert_eq!( @@ -96,7 +96,7 @@ fn alter_space() { with_variable("alter_space_test.global.db-tlog", |log_name| { let uuid; { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); uuid = init_space(&global, "myspace", "{}"); let stmt = lex_insecure("alter space myspace with { env: { SAYAN_MAX: 65536 } }".as_bytes()) @@ -105,7 +105,7 @@ fn alter_space() { Space::transactional_exec_alter(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); let spaces = global.state().idx().read(); let space = spaces.get("myspace").unwrap(); assert_eq!( @@ -125,14 +125,14 @@ fn alter_space() { fn drop_space() { with_variable("drop_space_test.global.db-tlog", |log_name| { { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); let _ = init_space(&global, "myspace", "{}"); let stmt = lex_insecure("drop space myspace".as_bytes()).unwrap(); let stmt = parse_ast_node_full(&stmt[2..]).unwrap(); Space::transactional_exec_drop(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); assert!(global.state().idx().read().get("myspace").is_none()); }) }) @@ -170,12 +170,12 @@ fn create_model() { let _uuid_space; let uuid_model; { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); _uuid_space = init_space(&global, "myspace", "{}"); uuid_model = init_default_model(&global); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global .state() .with_model(("myspace", "mymodel").into(), |model| { @@ -202,7 +202,7 @@ fn create_model() { fn alter_model_add() { with_variable("alter_model_add_test.global.db-tlog", |log_name| { { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); init_space(&global, "myspace", "{}"); init_default_model(&global); let stmt = lex_insecure( @@ -213,7 +213,7 @@ fn alter_model_add() { Model::transactional_exec_alter(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global .state() .with_model(("myspace", "mymodel").into(), |model| { @@ -232,7 +232,7 @@ fn alter_model_add() { fn alter_model_remove() { with_variable("alter_model_remove_test.global.db-tlog", |log_name| { { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); init_space(&global, "myspace", "{}"); init_model( &global, @@ -248,7 +248,7 @@ fn alter_model_remove() { Model::transactional_exec_alter(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global .state() .with_model(("myspace", "mymodel").into(), |model| { @@ -265,7 +265,7 @@ fn alter_model_remove() { fn alter_model_update() { with_variable("alter_model_update_test.global.db-tlog", |log_name| { { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); init_space(&global, "myspace", "{}"); init_model( &global, @@ -280,7 +280,7 @@ fn alter_model_update() { Model::transactional_exec_alter(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global .state() .with_model(("myspace", "mymodel").into(), |model| { @@ -299,7 +299,7 @@ fn alter_model_update() { fn drop_model() { with_variable("drop_model_test.global.db-tlog", |log_name| { { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); init_space(&global, "myspace", "{}"); init_default_model(&global); let stmt = lex_insecure(b"drop model myspace.mymodel").unwrap(); @@ -307,7 +307,7 @@ fn drop_model() { Model::transactional_exec_drop(&global, stmt).unwrap(); } multirun(|| { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); assert_eq!( global .state() diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs index 4e96f357..96a91f81 100644 --- a/server/src/engine/storage/mod.rs +++ b/server/src/engine/storage/mod.rs @@ -27,7 +27,6 @@ //! Implementations of the Skytable Disk Storage Subsystem (SDSS) use { - self::safe_interfaces::LocalFS, super::{ config::Configuration, core::GlobalNS, fractal::context, fractal::ModelDrivers, RuntimeResult, @@ -42,13 +41,8 @@ pub mod v1; pub mod v2; pub mod safe_interfaces { - #[cfg(test)] - pub use super::common::interface::fs_test::{NullFS, VirtualFS}; pub use super::{ - common::{ - interface::{fs_imp::LocalFS, fs_traits::FSInterface}, - paths_v1, - }, + common::{interface::fs::FileSystem, paths_v1}, v2::impls::mdl_journal::StdModelBatch, }; } @@ -64,8 +58,8 @@ pub use v2::impls::{ pub struct SELoaded { pub gns: GlobalNS, - pub gns_driver: v2::impls::gns_log::GNSDriver, - pub model_drivers: ModelDrivers, + pub gns_driver: v2::impls::gns_log::GNSDriver, + pub model_drivers: ModelDrivers, } pub fn load(cfg: &Configuration) -> RuntimeResult { diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs index bb1589bf..70d834c7 100644 --- a/server/src/engine/storage/v1/loader.rs +++ b/server/src/engine/storage/v1/loader.rs @@ -24,28 +24,27 @@ * */ -use std::collections::HashMap; - -use crate::engine::{ - core::{EntityIDRef, GlobalNS}, - error::RuntimeResult, - fractal::{error::ErrorContext, ModelUniqueID}, - storage::{ - common::{interface::fs_imp::LocalFS, paths_v1}, - v1::raw::{ - batch_jrnl, - journal::{raw as raw_journal, GNSAdapter}, - spec, +use { + crate::engine::{ + core::{EntityIDRef, GlobalNS}, + error::RuntimeResult, + fractal::{error::ErrorContext, ModelUniqueID}, + storage::{ + common::paths_v1, + v1::raw::{ + batch_jrnl, + journal::{raw as raw_journal, GNSAdapter}, + spec, + }, }, }, + std::collections::HashMap, }; pub fn load_gns() -> RuntimeResult { let gns = GlobalNS::empty(); - let gns_txn_driver = raw_journal::load_journal::( - super::GNS_PATH, - &gns, - )?; + let gns_txn_driver = + raw_journal::load_journal::(super::GNS_PATH, &gns)?; let mut model_drivers = HashMap::new(); let mut driver_guard = || { let mut models = gns.idx_models().write(); @@ -58,9 +57,9 @@ pub fn load_gns() -> RuntimeResult { .unwrap(); let path = paths_v1::model_path(space_name, space_uuid, model_name, model.get_uuid()); - let persist_driver = batch_jrnl::reinit::(&path, model).inherit_set_dmsg( - format!("failed to restore model data from journal in `{path}`"), - )?; + let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(format!( + "failed to restore model data from journal in `{path}`" + ))?; unsafe { // UNSAFE(@ohsayan): all pieces of data are upgraded by now, so vacuum model.model_mutator().vacuum_stashed(); diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs index 82c04189..ff08005d 100644 --- a/server/src/engine/storage/v1/mod.rs +++ b/server/src/engine/storage/v1/mod.rs @@ -33,7 +33,7 @@ pub mod raw; use { self::raw::sysdb::RestoredSystemDatabase, - super::common::interface::{fs_imp::LocalFS, fs_traits::FSInterface}, + super::common::interface::fs::FileSystem, crate::{ engine::{core::GlobalNS, RuntimeResult}, util, @@ -49,7 +49,7 @@ pub fn load_gns_prepare_migration() -> RuntimeResult { let gns = loader::load_gns()?; // load sysdb let RestoredSystemDatabase { users, .. } = - raw::sysdb::RestoredSystemDatabase::restore::(SYSDB_PATH)?; + raw::sysdb::RestoredSystemDatabase::restore(SYSDB_PATH)?; for (user, phash) in users { gns.sys_db().__raw_create_user(user, phash); } @@ -59,11 +59,11 @@ pub fn load_gns_prepare_migration() -> RuntimeResult { util::time_now_with_postfix("before_upgrade_to_v2") ); // move data folder - LocalFS::fs_create_dir_all(&backup_dir_path)?; + FileSystem::create_dir_all(&backup_dir_path)?; util::os::move_files_recursively("data", &format!("{backup_dir_path}/data"))?; // move GNS - LocalFS::fs_rename(GNS_PATH, &format!("{backup_dir_path}/{GNS_PATH}"))?; + FileSystem::rename(GNS_PATH, &format!("{backup_dir_path}/{GNS_PATH}"))?; // move sysdb - LocalFS::fs_rename(SYSDB_PATH, &format!("{backup_dir_path}/{SYSDB_PATH}"))?; + FileSystem::rename(SYSDB_PATH, &format!("{backup_dir_path}/{SYSDB_PATH}"))?; Ok(gns) } diff --git a/server/src/engine/storage/v1/raw/batch_jrnl/mod.rs b/server/src/engine/storage/v1/raw/batch_jrnl/mod.rs index d0f2a8b9..3e863236 100644 --- a/server/src/engine/storage/v1/raw/batch_jrnl/mod.rs +++ b/server/src/engine/storage/v1/raw/batch_jrnl/mod.rs @@ -42,20 +42,14 @@ pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver}; use { super::{rw::SDSSFileIO, spec}, - crate::engine::{ - core::model::Model, error::RuntimeResult, - storage::common::interface::fs_traits::FSInterface, - }, + crate::engine::{core::model::Model, error::RuntimeResult}, }; /// Re-initialize an existing batch journal and read all its data into model -pub fn reinit( - name: &str, - model: &Model, -) -> RuntimeResult> { - let (f, _header) = SDSSFileIO::::open::(name)?; +pub fn reinit(name: &str, model: &Model) -> RuntimeResult { + let (f, _header) = SDSSFileIO::open::(name)?; // restore let mut restore_driver = DataBatchRestoreDriver::new(f)?; restore_driver.read_data_batch_into_model(model)?; - DataBatchPersistDriver::new(restore_driver.into_file()?, false) + DataBatchPersistDriver::new(restore_driver.into_file(), false) } diff --git a/server/src/engine/storage/v1/raw/batch_jrnl/persist.rs b/server/src/engine/storage/v1/raw/batch_jrnl/persist.rs index 55120c19..0811406c 100644 --- a/server/src/engine/storage/v1/raw/batch_jrnl/persist.rs +++ b/server/src/engine/storage/v1/raw/batch_jrnl/persist.rs @@ -29,18 +29,18 @@ use { crate::engine::{ error::{RuntimeResult, StorageError}, storage::{ - common::interface::fs_traits::FSInterface, + common::interface::fs::File, v1::raw::rw::{SDSSFileIO, TrackedWriter}, }, }, }; -pub struct DataBatchPersistDriver { - f: TrackedWriter, +pub struct DataBatchPersistDriver { + f: TrackedWriter, } -impl DataBatchPersistDriver { - pub fn new(mut file: SDSSFileIO, is_new: bool) -> RuntimeResult { +impl DataBatchPersistDriver { + pub fn new(mut file: SDSSFileIO, is_new: bool) -> RuntimeResult { if !is_new { file.fsynced_write(&[MARKER_BATCH_REOPEN])?; } diff --git a/server/src/engine/storage/v1/raw/batch_jrnl/restore.rs b/server/src/engine/storage/v1/raw/batch_jrnl/restore.rs index 0116ee9b..48560d90 100644 --- a/server/src/engine/storage/v1/raw/batch_jrnl/restore.rs +++ b/server/src/engine/storage/v1/raw/batch_jrnl/restore.rs @@ -38,7 +38,7 @@ use { error::{RuntimeResult, StorageError}, idx::{MTIndex, STIndex, STIndexSeq}, storage::{ - common::interface::fs_traits::FSInterface, + common::interface::fs::File, common_encoding::r1::{ obj::cell::{self, StorageCellTypeID}, DataSource, @@ -105,17 +105,17 @@ enum Batch { BatchClosed, } -pub struct DataBatchRestoreDriver { - f: TrackedReader, +pub struct DataBatchRestoreDriver { + f: TrackedReader, } -impl DataBatchRestoreDriver { - pub fn new(f: SDSSFileIO) -> RuntimeResult { +impl DataBatchRestoreDriver { + pub fn new(f: SDSSFileIO) -> RuntimeResult { Ok(Self { - f: TrackedReader::new(f)?, + f: TrackedReader::new(f.into_buffered_reader())?, }) } - pub fn into_file(self) -> RuntimeResult> { + pub fn into_file(self) -> SDSSFileIO { self.f.into_inner_file() } pub(in crate::engine::storage::v1) fn read_data_batch_into_model( @@ -129,7 +129,7 @@ impl DataBatchRestoreDriver { } } -impl DataBatchRestoreDriver { +impl DataBatchRestoreDriver { fn read_all_batches_and_for_each( &mut self, mut f: impl FnMut(NormalBatch) -> RuntimeResult<()>, @@ -197,7 +197,7 @@ impl DataBatchRestoreDriver { } } -impl DataBatchRestoreDriver { +impl DataBatchRestoreDriver { fn apply_batch( m: &Model, NormalBatch { @@ -292,7 +292,7 @@ impl DataBatchRestoreDriver { } } -impl DataBatchRestoreDriver { +impl DataBatchRestoreDriver { fn read_batch_summary(&mut self, finished_early: bool) -> RuntimeResult { if !finished_early { // we must read the batch termination signature @@ -457,7 +457,7 @@ impl BatchStartBlock { } } -impl DataBatchRestoreDriver { +impl DataBatchRestoreDriver { fn decode_primary_key(&mut self, pk_type: u8) -> RuntimeResult { let Some(pk_type) = TagUnique::try_from_raw(pk_type) else { return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); @@ -495,7 +495,7 @@ impl DataBatchRestoreDriver { let Some(dscr) = StorageCellTypeID::try_from_raw(self.f.read_byte()?) else { return Err(StorageError::DataBatchRestoreCorruptedEntry.into()); }; - unsafe { cell::decode_element::>(&mut self.f, dscr) } + unsafe { cell::decode_element::(&mut self.f, dscr) } .map_err(|e| e.0) } } @@ -506,12 +506,18 @@ impl From for ErrorHack { Self(value) } } +impl From for ErrorHack { + fn from(value: std::io::Error) -> Self { + Self(value.into()) + } +} impl From<()> for ErrorHack { fn from(_: ()) -> Self { Self(StorageError::DataBatchRestoreCorruptedEntry.into()) } } -impl DataSource for TrackedReader { + +impl DataSource for TrackedReader { const RELIABLE_SOURCE: bool = false; type Error = ErrorHack; fn has_remaining(&self, cnt: usize) -> bool { diff --git a/server/src/engine/storage/v1/raw/journal/raw.rs b/server/src/engine/storage/v1/raw/journal/raw.rs index ca5a8678..1c0d42d7 100644 --- a/server/src/engine/storage/v1/raw/journal/raw.rs +++ b/server/src/engine/storage/v1/raw/journal/raw.rs @@ -46,7 +46,10 @@ use { crate::{ engine::{ error::{RuntimeResult, StorageError}, - storage::common::{interface::fs_traits::FSInterface, sdss}, + storage::common::{ + interface::fs::{BufferedReader, File}, + sdss, + }, }, util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy}, }, @@ -55,16 +58,12 @@ use { const CRC: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); -pub fn load_journal< - TA: JournalAdapter, - Fs: FSInterface, - F: sdss::sdss_r1::FileSpecV1, ->( +pub fn load_journal>( log_file_name: &str, gs: &TA::GlobalState, -) -> RuntimeResult> { - let (file, _) = SDSSFileIO::::open::(log_file_name)?; - let (file, last_txn_id) = JournalReader::::scroll(file, gs)?; +) -> RuntimeResult> { + let (file, _) = SDSSFileIO::open::(log_file_name)?; + let (file, last_txn_id) = JournalReader::::scroll(file, gs)?; JournalWriter::new(file, last_txn_id, false) } @@ -174,19 +173,19 @@ impl JournalEntryMetadata { } } -pub struct JournalReader { - log_file: SDSSFileIO, +pub struct JournalReader { + log_file: SDSSFileIO, evid: u64, closed: bool, remaining_bytes: u64, _m: PhantomData, } -impl JournalReader { - pub fn new(log_file: SDSSFileIO) -> RuntimeResult { +impl JournalReader { + pub fn new(log_file: SDSSFileIO) -> RuntimeResult { let log_size = log_file.file_length()? - Header::SIZE as u64; Ok(Self { - log_file, + log_file: log_file.into_buffered_reader(), evid: 0, closed: false, remaining_bytes: log_size, @@ -296,22 +295,22 @@ impl JournalReader { } /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID) pub fn scroll( - file: SDSSFileIO, + file: SDSSFileIO, gs: &TA::GlobalState, - ) -> RuntimeResult<(SDSSFileIO, u64)> { + ) -> RuntimeResult<(SDSSFileIO, u64)> { let mut slf = Self::new(file)?; while !slf.end_of_file() { slf.rapply_next_event(gs)?; } if slf.closed { - Ok((slf.log_file, slf.evid)) + Ok((slf.log_file.downgrade_reader(), slf.evid)) } else { Err(StorageError::JournalCorrupted.into()) } } } -impl JournalReader { +impl JournalReader { fn _incr_evid(&mut self) { self.evid += 1; } @@ -326,7 +325,7 @@ impl JournalReader { } } -impl JournalReader { +impl JournalReader { fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { if !self.has_remaining_bytes(buf.len() as _) { // do this right here to avoid another syscall @@ -338,17 +337,17 @@ impl JournalReader { } } -pub struct JournalWriter { +pub struct JournalWriter { /// the txn log file - log_file: SDSSFileIO, + log_file: SDSSFileIO, /// the id of the **next** journal id: u64, _m: PhantomData, closed: bool, } -impl JournalWriter { - pub fn new(mut log_file: SDSSFileIO, last_txn_id: u64, new: bool) -> RuntimeResult { +impl JournalWriter { + pub fn new(mut log_file: SDSSFileIO, last_txn_id: u64, new: bool) -> RuntimeResult { let log_size = log_file.file_length()?; log_file.seek_from_start(log_size)?; // avoid jumbling with headers let mut slf = Self { @@ -365,12 +364,12 @@ impl JournalWriter { } } -impl JournalWriter { +impl JournalWriter { pub fn append_journal_reopen(&mut self) -> RuntimeResult<()> { let id = self._incr_id() as u128; - self.log_file.fsynced_write( + e!(self.log_file.fsynced_write( &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(), - ) + )) } pub fn __close_mut(&mut self) -> RuntimeResult<()> { self.closed = true; @@ -385,7 +384,7 @@ impl JournalWriter { } } -impl JournalWriter { +impl JournalWriter { fn _incr_id(&mut self) -> u64 { let current = self.id; self.id += 1; @@ -393,7 +392,7 @@ impl JournalWriter { } } -impl Drop for JournalWriter { +impl Drop for JournalWriter { fn drop(&mut self) { assert!(self.closed, "log not closed"); } diff --git a/server/src/engine/storage/v1/raw/rw.rs b/server/src/engine/storage/v1/raw/rw.rs index e090e66b..dc979bfa 100644 --- a/server/src/engine/storage/v1/raw/rw.rs +++ b/server/src/engine/storage/v1/raw/rw.rs @@ -24,56 +24,52 @@ * */ -use { - crate::{ - engine::{ - error::RuntimeResult, - storage::common::{ - checksum::SCrc64, - interface::fs_traits::{ - FSInterface, FileInterface, FileInterfaceExt, FileInterfaceRead, - FileInterfaceWrite, FileInterfaceWriteExt, - }, - sdss, +use crate::{ + engine::{ + storage::common::{ + checksum::SCrc64, + interface::fs::{ + BufferedReader, BufferedWriter, File, FileExt, FileRead, FileWrite, FileWriteExt, }, + sdss, }, - util::os::SysIOError, + RuntimeResult, }, - std::marker::PhantomData, + util::os::SysIOError, + IoResult, }; -pub struct TrackedWriter { - file: SDSSFileIO::BufWriter>, +pub struct TrackedWriter { + file: SDSSFileIO, _cs: SCrc64, } -impl TrackedWriter { - pub fn new(f: SDSSFileIO) -> RuntimeResult { +impl TrackedWriter { + pub fn new(f: SDSSFileIO) -> IoResult { Ok(Self { - file: f.into_buffered_writer()?, + file: f.into_buffered_writer(), _cs: SCrc64::new(), }) } - pub fn sync_into_inner(self) -> RuntimeResult> { + pub fn sync_into_inner(self) -> IoResult> { self.file.downgrade_writer() } } /// [`SDSSFileLenTracked`] simply maintains application level length and checksum tracking to avoid frequent syscalls because we /// do not expect (even though it's very possible) users to randomly modify file lengths while we're reading them -pub struct TrackedReader { - f: SDSSFileIO::BufReader>, +pub struct TrackedReader { + f: SDSSFileIO, len: u64, cursor: u64, cs: SCrc64, } -impl TrackedReader { +impl TrackedReader { /// Important: this will only look at the data post the current cursor! - pub fn new(mut f: SDSSFileIO) -> RuntimeResult { + pub fn new(mut f: SDSSFileIO) -> IoResult { let len = f.file_length()?; let pos = f.file_cursor()?; - let f = f.into_buffered_reader()?; Ok(Self { f, len, @@ -90,10 +86,10 @@ impl TrackedReader { pub fn has_left(&self, v: u64) -> bool { self.remaining() >= v } - pub fn tracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + pub fn tracked_read(&mut self, buf: &mut [u8]) -> IoResult<()> { self.untracked_read(buf).map(|_| self.cs.update(buf)) } - pub fn read_byte(&mut self) -> RuntimeResult { + pub fn read_byte(&mut self) -> IoResult { let mut buf = [0u8; 1]; self.tracked_read(&mut buf).map(|_| buf[0]) } @@ -102,7 +98,7 @@ impl TrackedReader { core::mem::swap(&mut crc, &mut self.cs); crc.finish() } - pub fn untracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> { + pub fn untracked_read(&mut self, buf: &mut [u8]) -> IoResult<()> { if self.remaining() >= buf.len() as u64 { match self.f.read_buffer(buf) { Ok(()) => { @@ -112,94 +108,89 @@ impl TrackedReader { Err(e) => return Err(e), } } else { - Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()) + Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into_inner()) } } - pub fn into_inner_file(self) -> RuntimeResult> { + pub fn into_inner_file(self) -> SDSSFileIO { self.f.downgrade_reader() } - pub fn read_block(&mut self) -> RuntimeResult<[u8; N]> { + pub fn read_block(&mut self) -> IoResult<[u8; N]> { if !self.has_left(N as _) { - return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into()); + return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into_inner()); } let mut buf = [0; N]; self.tracked_read(&mut buf)?; Ok(buf) } - pub fn read_u64_le(&mut self) -> RuntimeResult { + pub fn read_u64_le(&mut self) -> IoResult { Ok(u64::from_le_bytes(self.read_block()?)) } } #[derive(Debug)] -pub struct SDSSFileIO::File> { +pub struct SDSSFileIO { f: F, - _fs: PhantomData, +} +impl SDSSFileIO { + pub fn new(f: F) -> Self { + Self { f } + } } -impl SDSSFileIO { - pub fn open>( +impl SDSSFileIO { + pub fn open>( fpath: &str, - ) -> RuntimeResult<(Self, F::Metadata)> { - let mut f = Self::_new(Fs::fs_fopen_rw(fpath)?); - let v = F::read_metadata(&mut f.f, ())?; + ) -> RuntimeResult<(Self, S::Metadata)> { + let mut f = Self::_new(File::open(fpath)?); + let v = S::read_metadata(&mut f.f, ())?; Ok((f, v)) } - pub fn into_buffered_reader( - self, - ) -> RuntimeResult::BufReader>> { - self.f.upgrade_to_buffered_reader().map(SDSSFileIO::_new) + pub fn into_buffered_reader(self) -> SDSSFileIO { + SDSSFileIO::new(self.f.into_buffered_reader()) } - pub fn into_buffered_writer( - self, - ) -> RuntimeResult::BufWriter>> { - self.f.upgrade_to_buffered_writer().map(SDSSFileIO::_new) + pub fn into_buffered_writer(self) -> SDSSFileIO { + SDSSFileIO::new(self.f.into_buffered_writer()) } } -impl SDSSFileIO::BufReader> { - pub fn downgrade_reader(self) -> RuntimeResult> { - let me = ::downgrade_reader(self.f)?; - Ok(SDSSFileIO::_new(me)) +impl SDSSFileIO { + pub fn downgrade_reader(self) -> SDSSFileIO { + SDSSFileIO::_new(self.f.into_inner()) } } -impl SDSSFileIO::BufWriter> { - pub fn downgrade_writer(self) -> RuntimeResult> { - let me = ::downgrade_writer(self.f)?; - Ok(SDSSFileIO::_new(me)) +impl SDSSFileIO { + pub fn downgrade_writer(self) -> IoResult> { + self.f.into_inner().map(SDSSFileIO::_new) } } -impl SDSSFileIO { +impl SDSSFileIO { fn _new(f: F) -> Self { - Self { - f, - _fs: PhantomData, - } + Self { f } } } -impl SDSSFileIO { - pub fn read_buffer(&mut self, buffer: &mut [u8]) -> RuntimeResult<()> { +impl SDSSFileIO { + pub fn read_buffer(&mut self, buffer: &mut [u8]) -> IoResult<()> { self.f.fread_exact(buffer) } } -impl SDSSFileIO { - pub fn file_cursor(&mut self) -> RuntimeResult { - self.f.fext_cursor() +impl SDSSFileIO { + pub fn file_cursor(&mut self) -> IoResult { + self.f.f_cursor() } - pub fn file_length(&self) -> RuntimeResult { - self.f.fext_length() + pub fn file_length(&self) -> IoResult { + self.f.f_len() } - pub fn seek_from_start(&mut self, by: u64) -> RuntimeResult<()> { - self.f.fext_seek_ahead_from_start_by(by) + pub fn seek_from_start(&mut self, by: u64) -> IoResult<()> { + self.f.f_seek_start(by) } } -impl SDSSFileIO { - pub fn read_full(&mut self) -> RuntimeResult> { +impl SDSSFileIO { + pub fn read_full(&mut self) -> IoResult> { let len = self.file_length()? - self.file_cursor()?; let mut buf = vec![0; len as usize]; self.read_buffer(&mut buf)?; @@ -207,9 +198,9 @@ impl SDSSFileIO } } -impl SDSSFileIO { - pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> { - self.f.fw_write_all(data)?; - self.f.fwext_sync_all() +impl SDSSFileIO { + pub fn fsynced_write(&mut self, data: &[u8]) -> IoResult<()> { + self.f.fwrite_all(data)?; + self.f.fsync_all() } } diff --git a/server/src/engine/storage/v1/raw/sysdb.rs b/server/src/engine/storage/v1/raw/sysdb.rs index 895e53f1..3fa16deb 100644 --- a/server/src/engine/storage/v1/raw/sysdb.rs +++ b/server/src/engine/storage/v1/raw/sysdb.rs @@ -30,9 +30,7 @@ use { core::system_db::SystemDatabase, data::{cell::Datacell, DictEntryGeneric, DictGeneric}, error::{RuntimeResult, StorageError}, - storage::{ - common::interface::fs_traits::FSInterface, common_encoding::r1, v1::raw::rw::SDSSFileIO, - }, + storage::{common_encoding::r1, v1::raw::rw::SDSSFileIO}, }, std::collections::HashMap, }; @@ -71,8 +69,8 @@ impl RestoredSystemDatabase { settings_version, } } - pub fn restore(name: &str) -> RuntimeResult { - let (mut f, _) = SDSSFileIO::::open::(name)?; + pub fn restore(name: &str) -> RuntimeResult { + let (mut f, _) = SDSSFileIO::open::(name)?; let mut sysdb_data = r1::dec::dict_full::(&f.read_full()?)?; // get our auth and sys stores let mut auth_store = rkey( diff --git a/server/src/engine/storage/v2/impls/gns_log.rs b/server/src/engine/storage/v2/impls/gns_log.rs index d7adec67..6b1d2e23 100644 --- a/server/src/engine/storage/v2/impls/gns_log.rs +++ b/server/src/engine/storage/v2/impls/gns_log.rs @@ -33,7 +33,6 @@ use { engine::{ core::GlobalNS, storage::{ - common::interface::fs_traits::FSInterface, common_encoding::r1::impls::gns::GNSEvent, v2::raw::journal::{self, EventLogDriver, JournalAdapterEvent}, }, @@ -56,19 +55,19 @@ use { GNS event log impl */ -pub type GNSDriver = EventLogDriver; +pub type GNSDriver = EventLogDriver; pub struct GNSEventLog; -impl GNSDriver { +impl GNSDriver { const FILE_PATH: &'static str = "gns.db-tlog"; pub fn open_gns_with_name(name: &str, gs: &GlobalNS) -> RuntimeResult { - journal::open_journal::<_, Fs>(name, gs) + journal::open_journal(name, gs) } pub fn open_gns(gs: &GlobalNS) -> RuntimeResult { Self::open_gns_with_name(Self::FILE_PATH, gs) } pub fn create_gns_with_name(name: &str) -> RuntimeResult { - journal::create_journal::<_, Fs>(name) + journal::create_journal(name) } /// Create a new event log pub fn create_gns() -> RuntimeResult { diff --git a/server/src/engine/storage/v2/impls/mdl_journal.rs b/server/src/engine/storage/v2/impls/mdl_journal.rs index 3f26fd79..d7128039 100644 --- a/server/src/engine/storage/v2/impls/mdl_journal.rs +++ b/server/src/engine/storage/v2/impls/mdl_journal.rs @@ -42,7 +42,7 @@ use { idx::{MTIndex, STIndex, STIndexSeq}, storage::{ common::{ - interface::fs_traits::{FSInterface, FileInterface}, + interface::fs::File, sdss::sdss_r1::rw::{TrackedReaderContext, TrackedWriter}, }, common_encoding::r1, @@ -67,14 +67,14 @@ use { }, }; -pub type ModelDriver = BatchDriver; -impl ModelDriver { +pub type ModelDriver = BatchDriver; +impl ModelDriver { pub fn open_model_driver(mdl: &Model, model_data_file_path: &str) -> RuntimeResult { - journal::open_journal::<_, Fs>(model_data_file_path, mdl) + journal::open_journal(model_data_file_path, mdl) } /// Create a new event log pub fn create_model_driver(model_data_file_path: &str) -> RuntimeResult { - journal::create_journal::<_, Fs>(model_data_file_path) + journal::create_journal(model_data_file_path) } } @@ -109,11 +109,11 @@ pub enum EventType { a little messy. */ -struct RowWriter<'b, Fs: FSInterface> { - f: &'b mut TrackedWriter as RawJournalAdapter>::Spec>, +struct RowWriter<'b> { + f: &'b mut TrackedWriter as RawJournalAdapter>::Spec>, } -impl<'b, Fs: FSInterface> RowWriter<'b, Fs> { +impl<'b> RowWriter<'b> { /// write global row information: /// - pk tag /// - schema version @@ -128,8 +128,9 @@ impl<'b, Fs: FSInterface> RowWriter<'b, Fs> { .value_u64() .u64_bytes_le(), )?; - self.f - .dtrack_write(&(model.fields().st_len() - 1).u64_bytes_le()) + e!(self + .f + .dtrack_write(&(model.fields().st_len() - 1).u64_bytes_le())) } /// write row metadata: /// - change type @@ -204,22 +205,19 @@ impl<'b, Fs: FSInterface> RowWriter<'b, Fs> { } } -struct BatchWriter<'a, 'b, Fs: FSInterface> { +struct BatchWriter<'a, 'b> { model: &'a Model, - row_writer: RowWriter<'b, Fs>, + row_writer: RowWriter<'b>, g: &'a Guard, sync_count: usize, } -impl<'a, 'b, Fs: FSInterface> BatchWriter<'a, 'b, Fs> { +impl<'a, 'b> BatchWriter<'a, 'b> { fn write_batch( model: &'a Model, g: &'a Guard, expected: usize, - f: &'b mut TrackedWriter< - Fs::File, - as RawJournalAdapter>::Spec, - >, + f: &'b mut TrackedWriter as RawJournalAdapter>::Spec>, batch_stat: &mut BatchStats, ) -> RuntimeResult { /* @@ -252,10 +250,7 @@ impl<'a, 'b, Fs: FSInterface> BatchWriter<'a, 'b, Fs> { fn new( model: &'a Model, g: &'a Guard, - f: &'b mut TrackedWriter< - Fs::File, - as RawJournalAdapter>::Spec, - >, + f: &'b mut TrackedWriter as RawJournalAdapter>::Spec>, ) -> RuntimeResult { let mut row_writer = RowWriter { f }; row_writer.write_row_global_metadata(model)?; @@ -310,10 +305,10 @@ impl<'a> JournalAdapterEvent> for StdModelBatch<' fn md(&self) -> u64 { BatchType::Standard.dscr_u64() } - fn write_direct( + fn write_direct( self, writer: &mut TrackedWriter< - Fs::File, + File, as RawJournalAdapter>::Spec, >, ctx: Rc>, @@ -322,12 +317,12 @@ impl<'a> JournalAdapterEvent> for StdModelBatch<' writer.dtrack_write(&self.1.u64_bytes_le())?; let g = pin(); let actual_commit = - BatchWriter::::write_batch(self.0, &g, self.1, writer, &mut ctx.borrow_mut())?; + BatchWriter::write_batch(self.0, &g, self.1, writer, &mut ctx.borrow_mut())?; if actual_commit != self.1 { // early exit writer.dtrack_write(&[EventType::EarlyExit.dscr()])?; } - writer.dtrack_write(&actual_commit.u64_bytes_le()) + e!(writer.dtrack_write(&actual_commit.u64_bytes_le())) } } @@ -343,16 +338,13 @@ impl<'a> JournalAdapterEvent> for FullModel<'a> { fn md(&self) -> u64 { BatchType::Standard.dscr_u64() } - fn write_direct( + fn write_direct( self, - f: &mut TrackedWriter< - Fs::File, - as RawJournalAdapter>::Spec, - >, + f: &mut TrackedWriter as RawJournalAdapter>::Spec>, _: Rc>, ) -> RuntimeResult<()> { let g = pin(); - let mut row_writer: RowWriter<'_, Fs> = RowWriter { f }; + let mut row_writer: RowWriter<'_> = RowWriter { f }; let index = self.0.primary_index().__raw_index(); let current_row_count = index.mt_len(); // expect commit == current row count @@ -452,21 +444,17 @@ impl BatchAdapterSpec for ModelDataAdapter { fn initialize_batch_state(_: &Self::GlobalState) -> Self::BatchState { BatchRestoreState { events: Vec::new() } } - fn decode_batch_metadata( + fn decode_batch_metadata( _: &Self::GlobalState, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + f: &mut TrackedReaderContext, batch_type: Self::BatchType, ) -> RuntimeResult { // [pk tag][schema version][column cnt] match batch_type { BatchType::Standard => {} } - let pk_tag = f.read_block().and_then(|[b]| { - TagUnique::try_from_raw(b).ok_or(StorageError::RawJournalCorrupted.into()) - })?; + let pk_tag = TagUnique::try_from_raw(f.read_block().map(|[b]| b)?) + .ok_or(StorageError::RawJournalCorrupted)?; let schema_version = u64::from_le_bytes(f.read_block()?); let column_count = u64::from_le_bytes(f.read_block()?); Ok(BatchMetadata { @@ -475,20 +463,17 @@ impl BatchAdapterSpec for ModelDataAdapter { column_count, }) } - fn update_state_for_new_event( + fn update_state_for_new_event( _: &Self::GlobalState, bs: &mut Self::BatchState, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + f: &mut TrackedReaderContext, batch_info: &Self::BatchMetadata, event_type: Self::EventType, ) -> RuntimeResult<()> { // get txn id let txn_id = u64::from_le_bytes(f.read_block()?); // get pk - let pk = restore_impls::decode_primary_key::(f, batch_info.pk_tag)?; + let pk = restore_impls::decode_primary_key::(f, batch_info.pk_tag)?; match event_type { EventType::Delete => { bs.events.push(DecodedBatchEvent::new( @@ -500,7 +485,7 @@ impl BatchAdapterSpec for ModelDataAdapter { EventType::Insert | EventType::Update => { // insert or update // prepare row - let row = restore_impls::decode_row_data::(batch_info, f)?; + let row = restore_impls::decode_row_data(batch_info, f)?; if event_type == EventType::Insert { bs.events.push(DecodedBatchEvent::new( txn_id, @@ -641,10 +626,7 @@ mod restore_impls { data::{cell::Datacell, tag::TagUnique}, error::StorageError, storage::{ - common::{ - interface::fs_traits::{FSInterface, FileInterface, FileInterfaceRead}, - sdss::sdss_r1::{rw::TrackedReaderContext, FileSpecV1}, - }, + common::sdss::sdss_r1::{rw::TrackedReaderContext, FileSpecV1}, common_encoding::r1::{ obj::cell::{self, StorageCellTypeID}, DataSource, @@ -658,8 +640,8 @@ mod restore_impls { /// Primary key decode impl /// /// NB: We really need to make this generic, but for now we can settle for this - pub fn decode_primary_key( - f: &mut TrackedReaderContext<<::File as FileInterface>::BufReader, S>, + pub fn decode_primary_key( + f: &mut TrackedReaderContext, pk_type: TagUnique, ) -> RuntimeResult { Ok(match pk_type { @@ -691,12 +673,9 @@ mod restore_impls { }, }) } - pub fn decode_row_data( + pub fn decode_row_data( batch_info: &BatchMetadata, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - ModelDataBatchAofV1, - >, + f: &mut TrackedReaderContext, ) -> Result, crate::engine::fractal::error::Error> { let mut row = vec![]; let mut this_col_cnt = batch_info.column_count; @@ -721,12 +700,17 @@ mod restore_impls { Self(value) } } + impl From for ErrorHack { + fn from(value: std::io::Error) -> Self { + Self(value.into()) + } + } impl From<()> for ErrorHack { fn from(_: ()) -> Self { Self(StorageError::DataBatchRestoreCorruptedEntry.into()) } } - impl<'a, F: FileInterfaceRead> DataSource for TrackedReaderContext<'a, F, ModelDataBatchAofV1> { + impl<'a> DataSource for TrackedReaderContext<'a, ModelDataBatchAofV1> { const RELIABLE_SOURCE: bool = false; type Error = ErrorHack; fn has_remaining(&self, cnt: usize) -> bool { diff --git a/server/src/engine/storage/v2/impls/tests/model_driver.rs b/server/src/engine/storage/v2/impls/tests/model_driver.rs index 435ca65d..2fb12abd 100644 --- a/server/src/engine/storage/v2/impls/tests/model_driver.rs +++ b/server/src/engine/storage/v2/impls/tests/model_driver.rs @@ -37,7 +37,6 @@ use { dml::{ins::InsertStatement, upd::UpdateStatement}, tests::lex_insecure, }, - storage::common::interface::fs_test::VirtualFS, }, util::test_utils, }, @@ -64,10 +63,7 @@ fn create_test_kv_int(change_count: usize) -> Vec<(u64, String)> { .collect() } -fn create_model_and_space( - global: &TestGlobal, - create_model: &str, -) -> QueryResult { +fn create_model_and_space(global: &TestGlobal, create_model: &str) -> QueryResult { let tokens = lex_insecure(create_model.as_bytes()).unwrap(); let create_model: CreateModel = ast::parse_ast_node_full(&tokens[2..]).unwrap(); let mdl_name = EntityID::new( @@ -82,13 +78,13 @@ fn create_model_and_space( Model::transactional_exec_create(global, create_model).map(|_| mdl_name) } -fn run_insert(global: &TestGlobal, insert: &str) -> QueryResult<()> { +fn run_insert(global: &TestGlobal, insert: &str) -> QueryResult<()> { let tokens = lex_insecure(insert.as_bytes()).unwrap(); let insert: InsertStatement = ast::parse_ast_node_full(&tokens[1..]).unwrap(); dml::insert(global, insert) } -fn run_update(global: &TestGlobal, update: &str) -> QueryResult<()> { +fn run_update(global: &TestGlobal, update: &str) -> QueryResult<()> { let tokens = lex_insecure(update.as_bytes()).unwrap(); let insert: UpdateStatement = ast::parse_ast_node_full(&tokens[1..]).unwrap(); dml::update(global, insert) @@ -110,12 +106,12 @@ fn create_and_close(log_name: &str, decl: &str) { test_utils::with_variable(log_name, |log_name| { // create and close { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); let _ = create_model_and_space(&global, decl).unwrap(); } // open { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); drop(global); } }) @@ -135,7 +131,7 @@ fn run_sample_inserts( // create, insert and close let mdl_name; { - let mut global = TestGlobal::new_with_vfs_driver(log_name); + let mut global = TestGlobal::new_with_driver_id(log_name); global.set_max_data_pressure(key_values.len()); mdl_name = create_model_and_space(&global, decl).unwrap(); for (username, password) in key_values.iter() { @@ -144,7 +140,7 @@ fn run_sample_inserts( } // reopen and verify 100 times test_utils::multi_run(100, || { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global.load_model_drivers().unwrap(); global .state() @@ -189,7 +185,7 @@ fn run_sample_updates( let mdl_name; { // insert n values - let mut global = TestGlobal::new_with_vfs_driver(log_name); + let mut global = TestGlobal::new_with_driver_id(log_name); global.set_max_data_pressure(n); mdl_name = create_model_and_space(&global, decl).unwrap(); for (username, password) in key_values.iter() { @@ -204,7 +200,7 @@ fn run_sample_updates( // now update values let mut actual_position = 0; for _ in 0..reopen_count { - let mut global = TestGlobal::new_with_vfs_driver(log_name); + let mut global = TestGlobal::new_with_driver_id(log_name); global.set_max_data_pressure(changes_per_cycle); global.load_model_drivers().unwrap(); let mut j = 0; @@ -220,7 +216,7 @@ fn run_sample_updates( assert_eq!(actual_position, n); } { - let global = TestGlobal::new_with_vfs_driver(log_name); + let global = TestGlobal::new_with_driver_id(log_name); global.load_model_drivers().unwrap(); for (txn_id, (username, password)) in key_values .iter() diff --git a/server/src/engine/storage/v2/mod.rs b/server/src/engine/storage/v2/mod.rs index 45859500..a93467c6 100644 --- a/server/src/engine/storage/v2/mod.rs +++ b/server/src/engine/storage/v2/mod.rs @@ -26,10 +26,7 @@ use { self::impls::mdl_journal::{BatchStats, FullModel}, - super::{ - common::interface::{fs_imp::LocalFS, fs_traits::FSInterface}, - v1, SELoaded, - }, + super::{common::interface::fs::FileSystem, v1, SELoaded}, crate::engine::{ config::Configuration, core::{ @@ -64,14 +61,14 @@ pub fn recreate(gns: GlobalNS) -> RuntimeResult { // create all spaces context::set_dmsg("creating all spaces"); for (space_name, space) in gns.idx().read().iter() { - LocalFS::fs_create_dir_all(&paths_v1::space_dir(space_name, space.get_uuid()))?; + FileSystem::create_dir_all(&paths_v1::space_dir(space_name, space.get_uuid()))?; gns_driver.commit_event(CreateSpaceTxn::new(space.props(), &space_name, space))?; } // create all models context::set_dmsg("creating all models"); for (model_id, model) in gns.idx_models().read().iter() { let space_uuid = gns.idx().read().get(model_id.space()).unwrap().get_uuid(); - LocalFS::fs_create_dir_all(&paths_v1::model_dir( + FileSystem::create_dir_all(&paths_v1::model_dir( model_id.space(), space_uuid, model_id.entity(), @@ -102,7 +99,7 @@ pub fn recreate(gns: GlobalNS) -> RuntimeResult { } pub fn initialize_new(config: &Configuration) -> RuntimeResult { - LocalFS::fs_create_dir_all(DATA_DIR)?; + FileSystem::create_dir_all(DATA_DIR)?; let mut gns_driver = impls::gns_log::GNSDriver::create_gns()?; let gns = GlobalNS::empty(); let password_hash = rcrypt::hash(&config.auth.root_key, rcrypt::DEFAULT_COST).unwrap(); diff --git a/server/src/engine/storage/v2/raw/journal/mod.rs b/server/src/engine/storage/v2/raw/journal/mod.rs index 59307f88..f01f7372 100644 --- a/server/src/engine/storage/v2/raw/journal/mod.rs +++ b/server/src/engine/storage/v2/raw/journal/mod.rs @@ -24,6 +24,8 @@ * */ +use crate::engine::storage::common::interface::fs::File; + use { self::raw::{CommitPreference, RawJournalAdapterEvent, RawJournalWriter}, crate::{ @@ -31,7 +33,6 @@ use { error::StorageError, storage::common::{ checksum::SCrc64, - interface::fs_traits::{FSInterface, FileInterface}, sdss::sdss_r1::{ rw::{TrackedReader, TrackedReaderContext, TrackedWriter}, FileSpecV1, @@ -62,7 +63,7 @@ pub use raw::{ */ /// An event log driver -pub type EventLogDriver = RawJournalWriter, Fs>; +pub type EventLogDriver = RawJournalWriter>; /// The event log adapter pub struct EventLogAdapter(PhantomData); type DispatchFn = fn(&G, Vec) -> RuntimeResult<()>; @@ -96,16 +97,13 @@ impl RawJournalAdapter for EventLogAdapter { fn initialize(_: &raw::JournalInitializer) -> Self { Self(PhantomData) } - fn enter_context<'a, Fs: FSInterface>( - _: &'a mut RawJournalWriter, - ) -> Self::Context<'a> { - } + fn enter_context<'a>(_: &'a mut RawJournalWriter) -> Self::Context<'a> {} fn parse_event_meta(meta: u64) -> Option { <::EventMeta as TaggedEnum>::try_from_raw(meta as u8) } - fn commit_direct<'a, Fs: FSInterface, E>( + fn commit_direct<'a, E>( &mut self, - w: &mut TrackedWriter, + w: &mut TrackedWriter, ev: E, ctx: (), ) -> RuntimeResult<()> @@ -124,15 +122,12 @@ impl RawJournalAdapter for EventLogAdapter { */ w.tracked_write(&checksum)?; w.tracked_write(&plen)?; - w.tracked_write(&pl) + e!(w.tracked_write(&pl)) } - fn decode_apply<'a, Fs: FSInterface>( + fn decode_apply<'a>( gs: &Self::GlobalState, meta: Self::EventMeta, - file: &mut TrackedReader< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + file: &mut TrackedReader, ) -> RuntimeResult<()> { let expected_checksum = u64::from_le_bytes(file.read_block()?); let plen = u64::from_le_bytes(file.read_block()?); @@ -163,31 +158,28 @@ impl RawJournalAdapter for EventLogAdapter { */ /// Batch journal driver -pub type BatchDriver = RawJournalWriter, Fs>; +pub type BatchDriver = RawJournalWriter>; /// Batch journal adapter pub struct BatchAdapter(PhantomData); #[cfg(test)] impl BatchAdapter { /// Open a new batch journal - pub fn open( - name: &str, - gs: &BA::GlobalState, - ) -> RuntimeResult> + pub fn open(name: &str, gs: &BA::GlobalState) -> RuntimeResult> where BA::Spec: FileSpecV1, { - raw::open_journal::, Fs>(name, gs) + raw::open_journal::>(name, gs) } /// Create a new batch journal - pub fn create(name: &str) -> RuntimeResult> + pub fn create(name: &str) -> RuntimeResult> where BA::Spec: FileSpecV1, { - raw::create_journal::, Fs>(name) + raw::create_journal::>(name) } /// Close a batch journal - pub fn close(me: &mut BatchDriver) -> RuntimeResult<()> { + pub fn close(me: &mut BatchDriver) -> RuntimeResult<()> { RawJournalWriter::close_driver(me) } } @@ -216,22 +208,16 @@ pub trait BatchAdapterSpec { /// initialize the batch state fn initialize_batch_state(gs: &Self::GlobalState) -> Self::BatchState; /// decode batch start metadata - fn decode_batch_metadata( + fn decode_batch_metadata( gs: &Self::GlobalState, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + f: &mut TrackedReaderContext, meta: Self::BatchType, ) -> RuntimeResult; /// decode new event and update state. if called, it is guaranteed that the event is not an early exit - fn update_state_for_new_event( + fn update_state_for_new_event( gs: &Self::GlobalState, bs: &mut Self::BatchState, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + f: &mut TrackedReaderContext, batch_info: &Self::BatchMetadata, event_type: Self::EventType, ) -> RuntimeResult<()>; @@ -253,30 +239,27 @@ impl RawJournalAdapter for BatchAdapter { fn initialize(_: &raw::JournalInitializer) -> Self { Self(PhantomData) } - fn enter_context<'a, Fs: FSInterface>( - _: &'a mut RawJournalWriter, - ) -> Self::Context<'a> { - } + fn enter_context<'a>(_: &'a mut RawJournalWriter) -> Self::Context<'a> {} fn parse_event_meta(meta: u64) -> Option { <::BatchType as TaggedEnum>::try_from_raw(meta as u8) } - fn commit_direct<'a, Fs: FSInterface, E>( + fn commit_direct<'a, E>( &mut self, - w: &mut TrackedWriter, + w: &mut TrackedWriter, ev: E, ctx: Self::CommitContext, ) -> RuntimeResult<()> where E: RawJournalAdapterEvent, { - ev.write_direct::(w, ctx)?; + ev.write_direct(w, ctx)?; let checksum = w.reset_partial(); - w.tracked_write(&checksum.to_le_bytes()) + e!(w.tracked_write(&checksum.to_le_bytes())) } - fn decode_apply<'a, Fs: FSInterface>( + fn decode_apply<'a>( gs: &Self::GlobalState, meta: Self::EventMeta, - f: &mut TrackedReader<<::File as FileInterface>::BufReader, Self::Spec>, + f: &mut TrackedReader, ) -> RuntimeResult<()> { let mut f = f.context(); { @@ -284,7 +267,7 @@ impl RawJournalAdapter for BatchAdapter { // read batch size let _stored_expected_commit_size = u64::from_le_bytes(f.read_block()?); // read custom metadata - let batch_md = ::decode_batch_metadata::(gs, &mut f, meta)?; + let batch_md = ::decode_batch_metadata(gs, &mut f, meta)?; // now read in every event let mut real_commit_size = 0; let mut batch_state = ::initialize_batch_state(gs); @@ -292,16 +275,16 @@ impl RawJournalAdapter for BatchAdapter { if real_commit_size == _stored_expected_commit_size { break; } - let event_type = f.read_block().and_then(|[b]| { - <::EventType as TaggedEnum>::try_from_raw(b) - .ok_or(StorageError::RawJournalCorrupted.into()) - })?; + let event_type = <::EventType as TaggedEnum>::try_from_raw( + f.read_block().map(|[b]| b)?, + ) + .ok_or(StorageError::RawJournalCorrupted)?; // is this an early exit marker? if so, exit if ::is_early_exit(&event_type) { break; } // update batch state - BA::update_state_for_new_event::( + BA::update_state_for_new_event( gs, &mut batch_state, &mut f, diff --git a/server/src/engine/storage/v2/raw/journal/raw/mod.rs b/server/src/engine/storage/v2/raw/journal/raw/mod.rs index 5a853b55..c049dde6 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/mod.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/mod.rs @@ -33,7 +33,7 @@ use { mem::unsafe_apis::memcpy, storage::common::{ checksum::SCrc64, - interface::fs_traits::{FSInterface, FileInterface}, + interface::fs::File, sdss::sdss_r1::{ rw::{SdssFile, TrackedReader, TrackedWriter}, FileSpecV1, @@ -49,13 +49,11 @@ use { */ /// Create a new journal -pub fn create_journal( - log_path: &str, -) -> RuntimeResult> +pub fn create_journal(log_path: &str) -> RuntimeResult> where J::Spec: FileSpecV1, { - let log = SdssFile::create::(log_path)?; + let log = SdssFile::create(log_path)?; RawJournalWriter::new( JournalInitializer::new(::SIZE as u64, SCrc64::new(), 0, 0), log, @@ -63,15 +61,15 @@ where } /// Open an existing journal -pub fn open_journal( +pub fn open_journal( log_path: &str, gs: &J::GlobalState, -) -> RuntimeResult> +) -> RuntimeResult> where J::Spec: FileSpecV1, { - let log = SdssFile::<_, J::Spec>::open::(log_path)?; - let (initializer, file) = RawJournalReader::::scroll(log, gs)?; + let log = SdssFile::::open(log_path)?; + let (initializer, file) = RawJournalReader::::scroll(log, gs)?; RawJournalWriter::new(initializer, file) } @@ -212,9 +210,9 @@ macro_rules! jtrace_reader { pub trait RawJournalAdapterEvent: Sized { fn md(&self) -> u64; - fn write_direct( + fn write_direct( self, - _: &mut TrackedWriter::Spec>, + _: &mut TrackedWriter::Spec>, _: ::CommitContext, ) -> RuntimeResult<()> { unimplemented!() @@ -247,15 +245,13 @@ pub trait RawJournalAdapter: Sized { /// initialize this adapter fn initialize(j_: &JournalInitializer) -> Self; /// get a write context - fn enter_context<'a, Fs: FSInterface>( - adapter: &'a mut RawJournalWriter, - ) -> Self::Context<'a>; + fn enter_context<'a>(adapter: &'a mut RawJournalWriter) -> Self::Context<'a>; /// parse event metadata fn parse_event_meta(meta: u64) -> Option; /// commit event (direct preference) - fn commit_direct( + fn commit_direct( &mut self, - _: &mut TrackedWriter, + _: &mut TrackedWriter, _: E, _: Self::CommitContext, ) -> RuntimeResult<()> @@ -272,13 +268,10 @@ pub trait RawJournalAdapter: Sized { unimplemented!() } /// decode and apply the event - fn decode_apply<'a, Fs: FSInterface>( + fn decode_apply<'a>( gs: &Self::GlobalState, meta: Self::EventMeta, - file: &mut TrackedReader< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + file: &mut TrackedReader, ) -> RuntimeResult<()>; } @@ -473,9 +466,9 @@ pub(super) enum DriverEventKind { */ /// A low-level journal writer -pub struct RawJournalWriter { +pub struct RawJournalWriter { j: J, - log_file: TrackedWriter<::File, ::Spec>, + log_file: TrackedWriter::Spec>, txn_id: u64, known_txn_id: u64, known_txn_offset: u64, // if offset is 0, txn id is unset @@ -483,9 +476,9 @@ pub struct RawJournalWriter { const SERVER_EV_MASK: u64 = 1 << (u64::BITS - 1); -impl RawJournalWriter { +impl RawJournalWriter { /// Initialize a new [`RawJournalWriter`] using a [`JournalInitializer`] - pub fn new(j_: JournalInitializer, file: SdssFile) -> RuntimeResult { + pub fn new(j_: JournalInitializer, file: SdssFile) -> RuntimeResult { let mut me = Self { log_file: TrackedWriter::with_cursor_and_checksum(file, j_.cursor(), j_.checksum()), known_txn_id: j_.last_txn_id(), @@ -533,7 +526,7 @@ impl RawJournalWriter { log_file.tracked_write(&ev_md.to_le_bytes())?; jtrace_writer!(CommitServerEventWroteMetadata); // now hand over control to adapter impl - J::commit_direct::(j, log_file, event, ctx)?; + J::commit_direct(j, log_file, event, ctx)?; } } jtrace_writer!(CommitServerEventAdapterCompleted); @@ -557,7 +550,7 @@ impl RawJournalWriter { } } -impl RawJournalWriter { +impl RawJournalWriter { fn txn_context( &mut self, f: impl FnOnce(&mut Self, u128) -> RuntimeResult, @@ -609,11 +602,8 @@ impl RawJournalWriter { } } -pub struct RawJournalReader { - tr: TrackedReader< - <::File as FileInterface>::BufReader, - ::Spec, - >, +pub struct RawJournalReader { + tr: TrackedReader<::Spec>, txn_id: u64, last_txn_id: u64, last_txn_offset: u64, @@ -636,14 +626,11 @@ impl JournalStats { } } -impl RawJournalReader { +impl RawJournalReader { pub fn scroll( - file: SdssFile<::File, ::Spec>, + file: SdssFile<::Spec>, gs: &J::GlobalState, - ) -> RuntimeResult<( - JournalInitializer, - SdssFile<::File, J::Spec>, - )> { + ) -> RuntimeResult<(JournalInitializer, SdssFile)> { let reader = TrackedReader::with_cursor( file, <::Spec as FileSpecV1>::SIZE as u64, @@ -660,16 +647,13 @@ impl RawJournalReader { // NB: the last txn offset is important because it indicates that the log is new me.last_txn_offset, ); - let file = me.tr.into_inner::()?; + let file = me.tr.into_inner(); return Ok((initializer, file)); } } } fn new( - reader: TrackedReader< - <::File as FileInterface>::BufReader, - ::Spec, - >, + reader: TrackedReader<::Spec>, txn_id: u64, last_txn_id: u64, last_txn_offset: u64, @@ -692,7 +676,7 @@ impl RawJournalReader { } } -impl RawJournalReader { +impl RawJournalReader { fn _apply_next_event_and_stop(&mut self, gs: &J::GlobalState) -> RuntimeResult { let txn_id = u128::from_le_bytes(self.tr.read_block()?); let meta = u64::from_le_bytes(self.tr.read_block()?); @@ -715,7 +699,7 @@ impl RawJournalReader { // now parse the actual event let Self { tr: reader, .. } = self; // we do not consider a parsed event a success signal; so we must actually apply it - match J::decode_apply::(gs, meta, reader) { + match J::decode_apply(gs, meta, reader) { Ok(()) => { jtrace_reader!(ServerEventAppliedSuccess); Self::__refresh_known_txn(self); diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests.rs b/server/src/engine/storage/v2/raw/journal/raw/tests.rs index ff804d6a..94d46221 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/tests.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/tests.rs @@ -33,13 +33,7 @@ use { error::StorageError, fractal::error::ErrorContext, storage::{ - common::{ - interface::{ - fs_test::VirtualFS, - fs_traits::{FSInterface, FileInterface}, - }, - sdss::sdss_r1::rw::TrackedReader, - }, + common::sdss::sdss_r1::rw::TrackedReader, v2::raw::{ journal::raw::{JournalReaderTraceEvent, JournalWriterTraceEvent}, spec::SystemDatabaseV1, @@ -75,22 +69,19 @@ impl SimpleDB { fn data(&self) -> std::cell::Ref<'_, Vec> { self.data.borrow() } - fn clear( - &mut self, - log: &mut RawJournalWriter, - ) -> RuntimeResult<()> { + fn clear(&mut self, log: &mut RawJournalWriter) -> RuntimeResult<()> { log.commit_event(DbEventClear)?; self.data.get_mut().clear(); Ok(()) } - fn pop(&mut self, log: &mut RawJournalWriter) -> RuntimeResult<()> { + fn pop(&mut self, log: &mut RawJournalWriter) -> RuntimeResult<()> { self.data.get_mut().pop().unwrap(); log.commit_event(DbEventPop)?; Ok(()) } fn push( &mut self, - log: &mut RawJournalWriter, + log: &mut RawJournalWriter, new: impl ToString, ) -> RuntimeResult<()> { let new = new.to_string(); @@ -155,9 +146,7 @@ impl RawJournalAdapter for SimpleDBJournal { fn initialize(_: &JournalInitializer) -> Self { Self } - fn enter_context<'a, Fs: FSInterface>( - _: &'a mut RawJournalWriter, - ) -> Self::Context<'a> { + fn enter_context<'a>(_: &'a mut RawJournalWriter) -> Self::Context<'a> { () } fn parse_event_meta(meta: u64) -> Option { @@ -176,13 +165,10 @@ impl RawJournalAdapter for SimpleDBJournal { ) { event.write_buffered(buf, ctx) } - fn decode_apply<'a, Fs: FSInterface>( + fn decode_apply<'a>( gs: &Self::GlobalState, meta: Self::EventMeta, - file: &mut TrackedReader< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + file: &mut TrackedReader, ) -> RuntimeResult<()> { match meta { EventMeta::NewKey => { @@ -212,7 +198,7 @@ fn journal_open_close() { const JOURNAL_NAME: &str = "journal_open_close"; { // new boot - let mut j = create_journal::(JOURNAL_NAME).unwrap(); + let mut j = create_journal::(JOURNAL_NAME).unwrap(); assert_eq!( super::obtain_trace(), intovec![JournalWriterTraceEvent::Initialized] @@ -233,8 +219,7 @@ fn journal_open_close() { } { // second boot - let mut j = - open_journal::(JOURNAL_NAME, &SimpleDB::new()).unwrap(); + let mut j = open_journal::(JOURNAL_NAME, &SimpleDB::new()).unwrap(); assert_eq!( super::obtain_trace(), intovec![ @@ -273,8 +258,7 @@ fn journal_open_close() { } { // third boot - let mut j = - open_journal::(JOURNAL_NAME, &SimpleDB::new()).unwrap(); + let mut j = open_journal::(JOURNAL_NAME, &SimpleDB::new()).unwrap(); assert_eq!( super::obtain_trace(), intovec![ @@ -328,7 +312,7 @@ fn journal_with_server_single_event() { { let mut db = SimpleDB::new(); // new boot - let mut j = create_journal::(JOURNAL_NAME).unwrap(); + let mut j = create_journal::(JOURNAL_NAME).unwrap(); db.push(&mut j, "hello world").unwrap(); RawJournalWriter::close_driver(&mut j).unwrap(); assert_eq!( @@ -352,7 +336,7 @@ fn journal_with_server_single_event() { { let db = SimpleDB::new(); // second boot - let mut j = open_journal::(JOURNAL_NAME, &db) + let mut j = open_journal::(JOURNAL_NAME, &db) .set_dmsg_fn(|| format!("{:?}", super::obtain_trace())) .unwrap(); assert_eq!(db.data().len(), 1); @@ -401,7 +385,7 @@ fn journal_with_server_single_event() { { // third boot let db = SimpleDB::new(); - let mut j = open_journal::(JOURNAL_NAME, &db).unwrap(); + let mut j = open_journal::(JOURNAL_NAME, &db).unwrap(); assert_eq!(db.data().len(), 1); assert_eq!(db.data()[0], "hello world"); assert_eq!( @@ -460,7 +444,7 @@ fn journal_with_server_single_event() { #[test] fn multi_boot() { { - let mut j = create_journal::("multiboot").unwrap(); + let mut j = create_journal::("multiboot").unwrap(); let mut db = SimpleDB::new(); db.push(&mut j, "key_a").unwrap(); db.push(&mut j, "key_b").unwrap(); @@ -469,7 +453,7 @@ fn multi_boot() { } { let mut db = SimpleDB::new(); - let mut j = open_journal::("multiboot", &db).unwrap(); + let mut j = open_journal::("multiboot", &db).unwrap(); assert_eq!(db.data().as_ref(), vec!["key_a".to_string()]); db.clear(&mut j).unwrap(); db.push(&mut j, "myfinkey").unwrap(); @@ -477,7 +461,7 @@ fn multi_boot() { } { let db = SimpleDB::new(); - let mut j = open_journal::("multiboot", &db).unwrap(); + let mut j = open_journal::("multiboot", &db).unwrap(); assert_eq!(db.data().as_ref(), vec!["myfinkey".to_string()]); RawJournalWriter::close_driver(&mut j).unwrap(); } diff --git a/server/src/engine/storage/v2/raw/journal/tests.rs b/server/src/engine/storage/v2/raw/journal/tests.rs index 8eaa2e66..c2dbb525 100644 --- a/server/src/engine/storage/v2/raw/journal/tests.rs +++ b/server/src/engine/storage/v2/raw/journal/tests.rs @@ -39,10 +39,7 @@ use { mem::unsafe_apis, storage::{ common::{ - interface::{ - fs_test::VirtualFS, - fs_traits::{FSInterface, FileInterface}, - }, + interface::fs::File, sdss::sdss_r1::rw::{TrackedReaderContext, TrackedWriter}, }, v2::raw::{ @@ -155,22 +152,18 @@ impl TestDB { fn _ref(&self) -> Ref> { self.data.borrow() } - fn push( - &self, - log: &mut EventLogDriver, - key: &str, - ) -> RuntimeResult<()> { + fn push(&self, log: &mut EventLogDriver, key: &str) -> RuntimeResult<()> { log.commit_event(EventPush(key))?; self._mut().push(key.into()); Ok(()) } - fn pop(&self, log: &mut EventLogDriver) -> RuntimeResult<()> { + fn pop(&self, log: &mut EventLogDriver) -> RuntimeResult<()> { assert!(!self._ref().is_empty()); log.commit_event(EventPop)?; self._mut().pop().unwrap(); Ok(()) } - fn clear(&self, log: &mut EventLogDriver) -> RuntimeResult<()> { + fn clear(&self, log: &mut EventLogDriver) -> RuntimeResult<()> { log.commit_event(EventClear)?; self._mut().clear(); Ok(()) @@ -179,7 +172,7 @@ impl TestDB { fn open_log() -> ( TestDB, - super::raw::RawJournalWriter, VirtualFS>, + super::raw::RawJournalWriter>, ) { let db = TestDB::default(); let log = open_journal("jrnl", &db).unwrap(); @@ -265,11 +258,7 @@ impl BatchDB { self.inner.borrow() } /// As soon as two changes occur, we sync to disk - fn push( - &self, - driver: &mut BatchDriver, - key: &str, - ) -> RuntimeResult<()> { + fn push(&self, driver: &mut BatchDriver, key: &str) -> RuntimeResult<()> { let mut me = self._mut(); me.data.push(key.into()); let changed = me.data.len() - me.last_flushed_at; @@ -293,10 +282,10 @@ impl<'a> RawJournalAdapterEvent> for BatchDBFlush<' fn md(&self) -> u64 { BatchType::GenericBatch.dscr_u64() } - fn write_direct( + fn write_direct( self, f: &mut TrackedWriter< - Fs::File, + File, as super::raw::RawJournalAdapter>::Spec, >, ctx: Rc>, @@ -348,23 +337,17 @@ impl BatchAdapterSpec for BatchDBAdapter { fn is_early_exit(ev: &Self::EventType) -> bool { BatchEventType::EarlyExit.eq(ev) } - fn decode_batch_metadata( + fn decode_batch_metadata( _: &Self::GlobalState, - _: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + _: &mut TrackedReaderContext, _: Self::BatchType, ) -> RuntimeResult { Ok(()) } - fn update_state_for_new_event( + fn update_state_for_new_event( _: &Self::GlobalState, bs: &mut Self::BatchState, - f: &mut TrackedReaderContext< - <::File as FileInterface>::BufReader, - Self::Spec, - >, + f: &mut TrackedReaderContext, _: &Self::BatchMetadata, event_type: Self::EventType, ) -> RuntimeResult<()> { @@ -421,8 +404,7 @@ fn batch_simple() { } { let db = BatchDB::new(); - let mut batch_drv = - BatchAdapter::::open::("mybatch", &db).unwrap(); + let mut batch_drv = BatchAdapter::::open("mybatch", &db).unwrap(); assert_eq!( db._ref().data, ["key1", "key2", "key3", "key4", "key5", "key6"]