simplest fully auto query construction and execution

main
Ziyang Hu 2 years ago
parent 341b94bbcd
commit d9b1be6273

@ -1,5 +1,5 @@
use crate::db::eval::{compare_tuple_by_keys, tuple_eval};
use crate::db::table::{ColId, TableId};
use crate::db::table::{ColId, TableId, TableInfo};
use crate::error::CozoError::LogicError;
use crate::error::Result;
use crate::relation::data::{DataKind, EMPTY_DATA};
@ -19,7 +19,7 @@ pub enum IteratorSlot<'a> {
impl<'a> Debug for IteratorSlot<'a> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
IteratorSlot::Dummy => write!(f, "DummyIterator"),
IteratorSlot::Dummy { .. } => write!(f, "DummyIterator"),
IteratorSlot::Reified(_) => write!(f, "BaseIterator"),
}
}
@ -42,35 +42,37 @@ impl<'a> IteratorSlot<'a> {
#[derive(Debug)]
pub enum ExecPlan<'a> {
NodeIt {
NodeItPlan {
it: IteratorSlot<'a>,
tid: u32,
info: TableInfo,
binding: String,
},
EdgeIt {
EdgeItPlan {
it: IteratorSlot<'a>,
tid: u32,
info: TableInfo,
binding: String,
},
EdgeKeyOnlyBwdIt {
EdgeKeyOnlyBwdItPlan {
it: IteratorSlot<'a>,
tid: u32,
info: TableInfo,
},
// EdgeBwdIt { it: IteratorPtr<'a>, sess: &'a Session<'a>, tid: u32 },
// IndexIt {it: ..}
KeySortedWithAssocIt {
KeySortedWithAssocItPlan {
main: Box<ExecPlan<'a>>,
associates: Vec<(u32, IteratorSlot<'a>)>,
},
CartesianProdIt {
CartesianProdItPlan {
left: Box<ExecPlan<'a>>,
right: Box<ExecPlan<'a>>,
},
MergeJoinIt {
MergeJoinItPlan {
left: Box<ExecPlan<'a>>,
right: Box<ExecPlan<'a>>,
left_keys: Vec<(TableId, ColId)>,
right_keys: Vec<(TableId, ColId)>,
},
OuterMergeJoinIt {
OuterMergeJoinItPlan {
left: Box<ExecPlan<'a>>,
right: Box<ExecPlan<'a>>,
left_keys: Vec<(TableId, ColId)>,
@ -80,39 +82,39 @@ pub enum ExecPlan<'a> {
left_len: (usize, usize),
right_len: (usize, usize),
},
KeyedUnionIt {
KeyedUnionItPlan {
left: Box<ExecPlan<'a>>,
right: Box<ExecPlan<'a>>,
},
KeyedDifferenceIt {
KeyedDifferenceItPlan {
left: Box<ExecPlan<'a>>,
right: Box<ExecPlan<'a>>,
},
FilterIt {
it: Box<ExecPlan<'a>>,
FilterItPlan {
source: Box<ExecPlan<'a>>,
filter: Value<'a>,
},
EvalIt {
it: Box<ExecPlan<'a>>,
keys: Vec<Value<'a>>,
vals: Vec<Value<'a>>,
out_prefix: u32,
EvalItPlan {
source: Box<ExecPlan<'a>>,
keys: Vec<(String, Value<'a>)>,
vals: Vec<(String, Value<'a>)>,
},
BagsUnionIt {
bags: Vec<ExecPlan<'a>>,
},
}
pub struct OutputIt<'a> {
it: ExecPlan<'a>,
filter: Value<'a>,
#[derive(Debug)]
pub struct OutputItPlan<'a> {
pub source: ExecPlan<'a>,
pub value: Value<'a>,
}
impl<'a> OutputIt<'a> {
impl<'a> OutputItPlan<'a> {
pub fn iter(&self) -> Result<OutputIterator> {
Ok(OutputIterator {
it: self.it.iter()?,
transform: &self.filter,
it: self.source.iter()?,
transform: &self.value,
})
}
}
@ -120,28 +122,28 @@ impl<'a> OutputIt<'a> {
impl<'a> ExecPlan<'a> {
pub fn iter(&'a self) -> Result<Box<dyn Iterator<Item=Result<MegaTuple>> + 'a>> {
match self {
ExecPlan::NodeIt { it, tid } => {
ExecPlan::NodeItPlan { it, info, .. } => {
let it = it.try_get()?;
let prefix_tuple = OwnTuple::with_prefix(*tid);
let prefix_tuple = OwnTuple::with_prefix(info.table_id.id as u32);
it.seek(prefix_tuple);
Ok(Box::new(NodeIterator { it, started: false }))
}
ExecPlan::EdgeIt { it, tid } => {
ExecPlan::EdgeItPlan { it, info, .. } => {
let it = it.try_get()?;
let prefix_tuple = OwnTuple::with_prefix(*tid);
let prefix_tuple = OwnTuple::with_prefix(info.table_id.id as u32);
it.seek(prefix_tuple);
Ok(Box::new(EdgeIterator { it, started: false }))
}
ExecPlan::EdgeKeyOnlyBwdIt { it, tid } => {
ExecPlan::EdgeKeyOnlyBwdItPlan { it, info } => {
let it = it.try_get()?;
let prefix_tuple = OwnTuple::with_prefix(*tid);
let prefix_tuple = OwnTuple::with_prefix(info.table_id.id as u32);
it.seek(prefix_tuple);
Ok(Box::new(EdgeKeyOnlyBwdIterator { it, started: false }))
}
ExecPlan::KeySortedWithAssocIt { main, associates } => {
ExecPlan::KeySortedWithAssocItPlan { main, associates } => {
let buffer = iter::repeat_with(|| None).take(associates.len()).collect();
let associates = associates
.iter()
@ -160,28 +162,26 @@ impl<'a> ExecPlan<'a> {
buffer,
}))
}
ExecPlan::CartesianProdIt { left, right } => Ok(Box::new(CartesianProdIterator {
ExecPlan::CartesianProdItPlan { left, right } => Ok(Box::new(CartesianProdIterator {
left: left.iter()?,
left_cache: MegaTuple::empty_tuple(),
right_source: right.as_ref(),
right: right.as_ref().iter()?,
})),
ExecPlan::FilterIt { it, filter } => Ok(Box::new(FilterIterator {
ExecPlan::FilterItPlan { source: it, filter } => Ok(Box::new(FilterIterator {
it: it.iter()?,
filter,
})),
ExecPlan::EvalIt {
it,
ExecPlan::EvalItPlan {
source: it,
keys,
vals,
out_prefix: prefix,
vals
} => Ok(Box::new(EvalIterator {
it: it.iter()?,
keys,
vals,
prefix: *prefix,
})),
ExecPlan::MergeJoinIt {
ExecPlan::MergeJoinItPlan {
left,
right,
left_keys,
@ -192,7 +192,7 @@ impl<'a> ExecPlan<'a> {
left_keys,
right_keys,
})),
ExecPlan::OuterMergeJoinIt {
ExecPlan::OuterMergeJoinItPlan {
left,
right,
left_keys,
@ -215,11 +215,11 @@ impl<'a> ExecPlan<'a> {
pull_left: true,
pull_right: true,
})),
ExecPlan::KeyedUnionIt { left, right } => Ok(Box::new(KeyedUnionIterator {
ExecPlan::KeyedUnionItPlan { left, right } => Ok(Box::new(KeyedUnionIterator {
left: left.iter()?,
right: right.iter()?,
})),
ExecPlan::KeyedDifferenceIt { left, right } => Ok(Box::new(KeyedDifferenceIterator {
ExecPlan::KeyedDifferenceItPlan { left, right } => Ok(Box::new(KeyedDifferenceIterator {
left: left.iter()?,
right: right.iter()?,
right_cache: None,
@ -864,11 +864,12 @@ impl<'a> Iterator for OutputIterator<'a> {
pub struct EvalIterator<'a> {
it: Box<dyn Iterator<Item=Result<MegaTuple>> + 'a>,
keys: &'a [Value<'a>],
vals: &'a [Value<'a>],
prefix: u32,
keys: &'a [(String, Value<'a>)],
vals: &'a [(String, Value<'a>)],
}
pub const EVAL_TEMP_PREFIX: u32 = u32::MAX - 1;
impl<'a> Iterator for EvalIterator<'a> {
type Item = Result<MegaTuple>;
@ -877,16 +878,16 @@ impl<'a> Iterator for EvalIterator<'a> {
None => None,
Some(Err(e)) => Some(Err(e)),
Some(Ok(t)) => {
let mut key_tuple = OwnTuple::with_prefix(self.prefix);
let mut key_tuple = OwnTuple::with_prefix(EVAL_TEMP_PREFIX);
let mut val_tuple = OwnTuple::with_data_prefix(DataKind::Data);
for k in self.keys {
match tuple_eval(k, &t) {
match tuple_eval(&k.1, &t) {
Ok(v) => key_tuple.push_value(&v),
Err(e) => return Some(Err(e)),
}
}
for k in self.vals {
match tuple_eval(k, &t) {
match tuple_eval(&k.1, &t) {
Ok(v) => val_tuple.push_value(&v),
Err(e) => return Some(Err(e)),
}
@ -973,13 +974,13 @@ mod tests {
.next()
.unwrap();
let sel_pat = sess.parse_select_pattern(p).unwrap();
let sel_vals = Value::Dict(sel_pat.vals.into_iter().map(|(k, v)| (k.into(), v)).collect());
let amap = sess.base_relation_to_accessor_map(
&from_pat.table,
&from_pat.binding,
&from_pat.info,
);
let (_, vals) = sess
.partial_eval(sel_pat.vals, &Default::default(), &amap)
.partial_eval(sel_vals, &Default::default(), &amap)
.unwrap();
let (_, where_vals) = sess
.partial_eval(where_pat, &Default::default(), &amap)
@ -1001,9 +1002,9 @@ mod tests {
let tbl = rel_tbls.pop().unwrap();
let it = sess.iter_node(tbl);
let it = ExecPlan::FilterIt {
let it = ExecPlan::FilterItPlan {
filter: where_vals,
it: it.into(),
source: it.into(),
};
let it = OutputIterator::new(&it, &vals)?;
for val in it {
@ -1012,7 +1013,7 @@ mod tests {
let duration = start.elapsed();
let duration2 = start2.elapsed();
println!("Time elapsed {:?} {:?}", duration, duration2);
let it = ExecPlan::KeySortedWithAssocIt {
let it = ExecPlan::KeySortedWithAssocItPlan {
main: Box::new(sess.iter_node(tbl)),
associates: vec![
(tbl.id as u32, sess.raw_iterator(true).into()),
@ -1032,8 +1033,8 @@ mod tests {
}
}
let mut it = sess.iter_node(tbl);
for _ in 0..3 {
it = ExecPlan::CartesianProdIt {
for _ in 0..2 {
it = ExecPlan::CartesianProdItPlan {
left: Box::new(it),
right: Box::new(sess.iter_node(tbl)),
}
@ -1065,9 +1066,21 @@ mod tests {
"{} items per second",
1e9 * (n as f64) / (duration.as_nanos() as f64)
);
// let a = sess.iter_table(tbl);
// let ac = (&a).into_iter().count();
// println!("{}", ac);
let s = r##"from e:Employee
where e.id >= 100, e.id <= 105 || e.id == 110
select {id: e.id,
full_name: e.first_name ++ ' ' ++ e.last_name, bibio_name: e.last_name ++ ', '
++ e.first_name ++ ': ' ++ (e.phone_number ~ 'N.A.')}"##;
let parsed = Parser::parse(Rule::relational_query, s)?.next().unwrap();
let plan = sess.query_to_plan(parsed)?;
println!("{:?}", plan);
let plan = sess.reify_output_plan(plan)?;
println!("{:?}", plan);
for val in plan.iter()? {
println!("{}", val?)
}
}
drop(engine);
let _ = fs::remove_dir_all(db_path);

@ -1,5 +1,5 @@
use crate::db::engine::Session;
use crate::db::iterator::{ExecPlan, IteratorSlot, OutputIt};
use crate::db::iterator::{ExecPlan, IteratorSlot, OutputItPlan};
use crate::db::query::{FromEl, Selection};
use crate::db::table::{ColId, TableId, TableInfo};
use crate::error::Result;
@ -8,6 +8,8 @@ use crate::relation::value::{StaticValue, Value};
use cozorocks::IteratorPtr;
use pest::iterators::Pair;
use std::collections::BTreeMap;
use crate::error::CozoError::LogicError;
use crate::relation::data::DataKind;
#[derive(Eq, PartialEq, Copy, Clone, Debug)]
pub enum OuterJoinType {
@ -19,13 +21,92 @@ pub enum OuterJoinType {
pub type AccessorMap = BTreeMap<String, BTreeMap<String, (TableId, ColId)>>;
impl<'a> Session<'a> {
pub fn realize_intermediate_plan(&self, _plan: ExecPlan) -> ExecPlan {
todo!()
pub fn reify_intermediate_plan(&'a self, plan: ExecPlan<'a>) -> Result<ExecPlan<'a>> {
self.do_reify_intermediate_plan(plan).map(|v| v.0)
}
pub fn realize_output_plan(&self, _plan: ExecPlan) -> OutputIt {
todo!()
fn convert_to_relative_amap(&self, amap: AccessorMap) -> AccessorMap {
// TODO this only handles the simplest case
fn convert_inner(inner: BTreeMap<String, (TableId, ColId)>) -> BTreeMap<String, (TableId, ColId)> {
inner.into_iter().map(|(k, (_tid, cid))| {
(k, (TableId::new(false, 0), cid))
}).collect()
}
amap.into_iter().map(|(k, v)| (k, convert_inner(v))).collect()
}
fn do_reify_intermediate_plan(&'a self, plan: ExecPlan<'a>) -> Result<(ExecPlan<'a>, AccessorMap)> {
let res = match plan {
ExecPlan::NodeItPlan { info, binding, .. } => {
let amap = self.base_relation_to_accessor_map(&binding, &info);
let amap = self.convert_to_relative_amap(amap);
let it = if info.table_id.in_root {
self.txn.iterator(true, &self.perm_cf)
} else {
self.txn.iterator(true, &self.temp_cf)
};
let it = IteratorSlot::Reified(it);
let plan = ExecPlan::NodeItPlan {
it,
info,
binding,
};
(plan, amap)
}
ExecPlan::EdgeItPlan { .. } => todo!(),
ExecPlan::EdgeKeyOnlyBwdItPlan { .. } => todo!(),
ExecPlan::KeySortedWithAssocItPlan { .. } => todo!(),
ExecPlan::CartesianProdItPlan { .. } => todo!(),
ExecPlan::MergeJoinItPlan { .. } => todo!(),
ExecPlan::OuterMergeJoinItPlan { .. } => todo!(),
ExecPlan::KeyedUnionItPlan { .. } => todo!(),
ExecPlan::KeyedDifferenceItPlan { .. } => todo!(),
ExecPlan::FilterItPlan { source: it, filter } => {
let (inner, amap) = self.do_reify_intermediate_plan(*it)?;
let (_, filter) = self.partial_eval(filter, &Default::default(), &amap)?;
let plan = ExecPlan::FilterItPlan {
source: inner.into(),
filter,
};
(plan, amap)
}
ExecPlan::EvalItPlan { source: it, keys, vals } => {
let (inner, amap) = self.do_reify_intermediate_plan(*it)?;
let keys = keys.into_iter().map(|(k, v)| -> Result<_> {
let (_, v) = self.partial_eval(v, &Default::default(), &amap)?;
Ok((k, v))
}).collect::<Result<Vec<_>>>()?;
let vals = vals.into_iter().map(|(k, v)| -> Result<_> {
let (_, v) = self.partial_eval(v, &Default::default(), &amap)?;
Ok((k, v))
}).collect::<Result<Vec<_>>>()?;
let plan = ExecPlan::EvalItPlan {
source: inner.into(),
keys,
vals,
};
(plan, amap)
}
ExecPlan::BagsUnionIt { .. } => todo!(),
};
Ok(res)
}
pub fn reify_output_plan(&'a self, plan: ExecPlan<'a>) -> Result<OutputItPlan<'a>> {
let plan = self.reify_intermediate_plan(plan)?;
let plan = match plan {
ExecPlan::EvalItPlan { source: it, mut keys, vals } => {
keys.extend(vals);
let filter = Value::Dict(keys.into_iter().map(|(k, v)| (k.into(), v)).collect());
OutputItPlan {
source: *it,
value: filter,
}
}
_plan => {
todo!()
}
};
Ok(plan)
}
pub fn query_to_plan(&self, pair: Pair<Rule>) -> Result<()> {
pub fn query_to_plan(&self, pair: Pair<Rule>) -> Result<ExecPlan> {
let mut pairs = pair.into_inner();
let from_data = self.parse_from_pattern(pairs.next().unwrap())?;
let mut nxt = pairs.next().unwrap();
@ -40,32 +121,39 @@ impl<'a> Session<'a> {
let select_data = self.parse_select_pattern(nxt)?;
let plan = self.convert_from_data_to_plan(from_data)?;
let plan = self.convert_where_data_to_plan(plan, where_data)?;
let plan = self.convert_select_data_to_plan(plan, select_data)?;
println!("{:#?}", plan);
Ok(())
self.convert_select_data_to_plan(plan, select_data)
}
fn convert_from_data_to_plan(&self, mut from_data: Vec<FromEl>) -> Result<ExecPlan> {
let _res = match from_data.pop().unwrap() {
let res = match from_data.pop().unwrap() {
FromEl::Simple(el) => {
println!(
"{:#?}",
self.base_relation_to_accessor_map(&el.table, &el.binding, &el.info)
);
todo!()
// QueryPlan::BaseRelation {
// table: el.table,
// binding: el.binding,
// info: el.info,
// }
// println!(
// "{:#?}",
// self.base_relation_to_accessor_map(&el.table, &el.binding, &el.info)
// );
match el.info.kind {
DataKind::Node => {
ExecPlan::NodeItPlan {
it: IteratorSlot::Dummy,
info: el.info,
binding: el.binding,
}
}
DataKind::Edge => {
ExecPlan::EdgeItPlan {
it: IteratorSlot::Dummy,
info: el.info,
binding: el.binding,
}
}
_ => return Err(LogicError("Wrong type for table binding".to_string()))
}
}
FromEl::Chain(_) => todo!(),
};
// Ok(res)
// todo!()
Ok(res)
}
pub(crate) fn base_relation_to_accessor_map(
&self,
_table: &str,
binding: &str,
info: &TableInfo,
) -> AccessorMap {
@ -86,29 +174,33 @@ impl<'a> Session<'a> {
}
BTreeMap::from([(binding.to_string(), ret)])
}
fn convert_where_data_to_plan(
fn convert_where_data_to_plan<'b>(
&self,
plan: ExecPlan,
plan: ExecPlan<'b>,
where_data: StaticValue,
) -> Result<ExecPlan> {
) -> Result<ExecPlan<'b>> {
let where_data = self.partial_eval(where_data, &Default::default(), &Default::default());
let _plan = match where_data?.1 {
let plan = match where_data?.1 {
Value::Bool(true) => plan,
_v => {
todo!()
// QueryPlan::Filter { rel: Box::new(plan), filter: v }
v => {
ExecPlan::FilterItPlan {
source: Box::new(plan),
filter: v,
}
}
};
// Ok(plan)
todo!()
Ok(plan)
}
fn convert_select_data_to_plan(
fn convert_select_data_to_plan<'b>(
&self,
_plan: ExecPlan,
_select_data: Selection,
) -> Result<ExecPlan> {
// Ok(MegaTupleIt::Projection { arg: Box::new(plan), projection: select_data })
todo!()
plan: ExecPlan<'b>,
select_data: Selection,
) -> Result<ExecPlan<'b>> {
Ok(ExecPlan::EvalItPlan {
source: Box::new(plan),
keys: select_data.keys,
vals: select_data.vals,
})
}
pub fn raw_iterator(&self, in_root: bool) -> IteratorPtr {
@ -119,11 +211,30 @@ impl<'a> Session<'a> {
}
}
// internal temp use
pub fn iter_node(&self, tid: TableId) -> ExecPlan {
let it = self.raw_iterator(tid.in_root);
ExecPlan::NodeIt {
ExecPlan::NodeItPlan {
it: IteratorSlot::Reified(it),
tid: tid.id as u32,
info: TableInfo {
kind: DataKind::Data,
table_id: tid,
src_table_id: Default::default(),
dst_table_id: Default::default(),
data_keys: Default::default(),
key_typing: vec![],
val_typing: vec![],
src_key_typing: vec![],
dst_key_typing: vec![],
associates: vec![],
},
binding: "".to_string(),
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn from_data() {}
}

@ -199,7 +199,7 @@ impl<'a> Session<'a> {
_ => None,
};
let mut keys = BTreeMap::new();
let mut keys = vec![];
let mut merged = vec![];
let mut collected_vals = BTreeMap::new();
@ -209,7 +209,7 @@ impl<'a> Session<'a> {
let mut pp = p.into_inner();
let id = parse_string(pp.next().unwrap())?;
let val = Value::from_pair(pp.next().unwrap())?;
keys.insert(id, val.to_static());
keys.push((id, val.to_static()));
}
Rule::dict_pair => {
let mut inner = p.into_inner();
@ -248,12 +248,14 @@ impl<'a> Session<'a> {
}
let vals = if merged.is_empty() {
Value::Dict(collected_vals).to_static()
collected_vals.into_iter().map(|(k, v)| (k.to_string(), v.to_static())).collect::<Vec<_>>()
} else {
if !collected_vals.is_empty() {
merged.push(Value::Dict(collected_vals));
}
Value::Apply(value::METHOD_MERGE.into(), merged).to_static()
// construct it with help of partial eval
todo!()
// if !collected_vals.is_empty() {
// merged.push(Value::Dict(collected_vals));
// }
// Value::Apply(value::METHOD_MERGE.into(), merged).to_static()
};
let mut ordering = vec![];
@ -315,8 +317,8 @@ impl<'a> Session<'a> {
#[derive(Debug, Eq, PartialEq, Clone)]
pub struct Selection {
pub scoped: Option<String>,
pub keys: BTreeMap<String, StaticValue>,
pub vals: StaticValue,
pub keys: Vec<(String, StaticValue)>,
pub vals: Vec<(String, StaticValue)>,
pub ordering: Vec<(bool, String)>,
pub limit: Option<i64>,
pub offset: Option<i64>,

Loading…
Cancel
Save