move stuff around; document features

main
Ziyang Hu 2 years ago
parent 9c3c97c32a
commit a863dd0de1

113
Cargo.lock generated

@ -250,9 +250,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]] [[package]]
name = "bytemuck" name = "bytemuck"
version = "1.12.2" version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aec14f5d4e6e3f927cd0c81f72e5710d95ee9019fbeb4b3021193867491bfd8" checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
@ -296,9 +296,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.74" version = "1.0.76"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
dependencies = [ dependencies = [
"jobserver", "jobserver",
] ]
@ -332,9 +332,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]] [[package]]
name = "chrono" name = "chrono"
version = "0.4.22" version = "0.4.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
dependencies = [ dependencies = [
"iana-time-zone", "iana-time-zone",
"js-sys", "js-sys",
@ -478,9 +478,9 @@ dependencies = [
"casey", "casey",
"chrono", "chrono",
"chrono-tz", "chrono-tz",
"clap",
"cozorocks", "cozorocks",
"csv", "csv",
"document-features",
"either", "either",
"env_logger", "env_logger",
"itertools 0.10.5", "itertools 0.10.5",
@ -500,7 +500,6 @@ dependencies = [
"rmp", "rmp",
"rmp-serde", "rmp-serde",
"rmpv", "rmpv",
"rouille",
"serde", "serde",
"serde_bytes", "serde_bytes",
"serde_derive", "serde_derive",
@ -550,6 +549,15 @@ dependencies = [
"zstd-sys", "zstd-sys",
] ]
[[package]]
name = "cozoserver"
version = "0.1.0"
dependencies = [
"clap",
"cozo",
"rouille",
]
[[package]] [[package]]
name = "cpufeatures" name = "cpufeatures"
version = "0.2.5" version = "0.2.5"
@ -645,9 +653,9 @@ dependencies = [
[[package]] [[package]]
name = "cxx" name = "cxx"
version = "1.0.80" version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" checksum = "97abf9f0eca9e52b7f81b945524e76710e6cb2366aead23b7d4fbf72e281f888"
dependencies = [ dependencies = [
"cc", "cc",
"cxxbridge-flags", "cxxbridge-flags",
@ -657,9 +665,9 @@ dependencies = [
[[package]] [[package]]
name = "cxx-build" name = "cxx-build"
version = "1.0.80" version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" checksum = "7cc32cc5fea1d894b77d269ddb9f192110069a8a9c1f1d441195fba90553dea3"
dependencies = [ dependencies = [
"cc", "cc",
"codespan-reporting", "codespan-reporting",
@ -672,15 +680,15 @@ dependencies = [
[[package]] [[package]]
name = "cxxbridge-flags" name = "cxxbridge-flags"
version = "1.0.80" version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" checksum = "8ca220e4794c934dc6b1207c3b42856ad4c302f2df1712e9f8d2eec5afaacf1f"
[[package]] [[package]]
name = "cxxbridge-macro" name = "cxxbridge-macro"
version = "1.0.80" version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" checksum = "b846f081361125bfc8dc9d3940c84e1fd83ba54bbca7b17cd29483c828be0704"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -753,6 +761,15 @@ dependencies = [
"crypto-common", "crypto-common",
] ]
[[package]]
name = "document-features"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3267e1ade4f1f6ddd35fed44a04b6514e244ffeda90c6a14a9ee30f9c9fd7a1"
dependencies = [
"litrs",
]
[[package]] [[package]]
name = "either" name = "either"
version = "1.8.0" version = "1.8.0"
@ -770,9 +787,9 @@ dependencies = [
[[package]] [[package]]
name = "env_logger" name = "env_logger"
version = "0.9.1" version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
dependencies = [ dependencies = [
"atty", "atty",
"humantime", "humantime",
@ -1424,6 +1441,12 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "litrs"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9275e0933cf8bb20f008924c0cb07a0692fe54d8064996520bf998de9eb79aa"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
version = "0.4.9" version = "0.4.9"
@ -1678,9 +1701,9 @@ dependencies = [
[[package]] [[package]]
name = "num_cpus" name = "num_cpus"
version = "1.13.1" version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"libc", "libc",
@ -1767,18 +1790,18 @@ dependencies = [
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "3.3.0" version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f74e330193f90ec45e2b257fa3ef6df087784157ac1ad2c1e71c62837b03aa7" checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf"
dependencies = [ dependencies = [
"num-traits", "num-traits",
] ]
[[package]] [[package]]
name = "os_str_bytes" name = "os_str_bytes"
version = "6.3.1" version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e"
[[package]] [[package]]
name = "owo-colors" name = "owo-colors"
@ -1840,9 +1863,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" checksum = "a528564cc62c19a7acac4d81e01f39e53e25e17b934878f4c6d25cc2836e62f8"
dependencies = [ dependencies = [
"thiserror", "thiserror",
"ucd-trie", "ucd-trie",
@ -1850,9 +1873,9 @@ dependencies = [
[[package]] [[package]]
name = "pest_derive" name = "pest_derive"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2" checksum = "d5fd9bc6500181952d34bd0b2b0163a54d794227b498be0b7afa7698d0a7b18f"
dependencies = [ dependencies = [
"pest", "pest",
"pest_generator", "pest_generator",
@ -1860,9 +1883,9 @@ dependencies = [
[[package]] [[package]]
name = "pest_generator" name = "pest_generator"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db" checksum = "d2610d5ac5156217b4ff8e46ddcef7cdf44b273da2ac5bca2ecbfa86a330e7c4"
dependencies = [ dependencies = [
"pest", "pest",
"pest_meta", "pest_meta",
@ -1873,9 +1896,9 @@ dependencies = [
[[package]] [[package]]
name = "pest_meta" name = "pest_meta"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d" checksum = "824749bf7e21dd66b36fbe26b3f45c713879cccd4a009a917ab8e045ca8246fe"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"pest", "pest",
@ -1961,9 +1984,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.16" version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]] [[package]]
name = "prettyplease" name = "prettyplease"
@ -1977,9 +2000,9 @@ dependencies = [
[[package]] [[package]]
name = "priority-queue" name = "priority-queue"
version = "1.2.3" version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "815082d99af3acc75a3e67efd2a07f72e67b4e81b4344eb8ca34c6ebf3dfa9c5" checksum = "d7685ca4cc0b3ad748c22ce6803e23b55b9206ef7715b965ebeaf41639238fdc"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"indexmap", "indexmap",
@ -2303,9 +2326,9 @@ dependencies = [
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.6.0" version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -2320,9 +2343,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
[[package]] [[package]]
name = "regex-syntax" name = "regex-syntax"
version = "0.6.27" version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]] [[package]]
name = "remove_dir_all" name = "remove_dir_all"
@ -2464,7 +2487,7 @@ dependencies = [
"serde_json", "serde_json",
"sha1", "sha1",
"threadpool", "threadpool",
"time 0.3.16", "time 0.3.17",
"tiny_http", "tiny_http",
"url", "url",
] ]
@ -2784,9 +2807,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]] [[package]]
name = "supports-color" name = "supports-color"
version = "1.3.0" version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4872ced36b91d47bae8a214a683fe54e7078875b399dfa251df346c9b547d1f9" checksum = "8ba6faf2ca7ee42fdd458f4347ae0a9bd6bcc445ad7cb57ad82b383f18870d6f"
dependencies = [ dependencies = [
"atty", "atty",
"is_ci", "is_ci",
@ -3030,9 +3053,9 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.16" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
dependencies = [ dependencies = [
"libc", "libc",
"num_threads", "num_threads",

@ -2,4 +2,4 @@
lto = true lto = true
[workspace] [workspace]
members = ["cozorocks", "cozo-lib-c", "cozo-lib-java", "cozo-core"] members = ["cozorocks", "cozo-lib-c", "cozo-lib-java", "cozo-core", "cozoserver"]

@ -16,12 +16,32 @@ exclude = [
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features] [features]
storage-rocksdb = [] #! # Features
storage-sled = []
storage-tikv = [] default = ["compact", "storage-rocksdb"]
#storage-redb = [] ## Enables the `minimal`, `requests` and `graph-algo` features
jemalloc = ["tikv-jemallocator-global", "cozorocks/jemalloc"] compact = ["minimal", "requests", "graph-algo"]
io-uring = ["cozorocks/io-uring"] ## Enables the `storage-sqlite` feature
minimal = ["storage-sqlite"]
## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files.
storage-sqlite = ["dep:sqlite"]
## Enables the [RocksDB](http://rocksdb.org/) backend
storage-rocksdb = ["dep:cozorocks"]
## Enables the graph algorithms
graph-algo = ["dep:rayon", "dep:nalgebra"]
## Allows the utilities to make web requests to fetch data
requests = ["dep:minreq"]
## Uses jemalloc as the global allocator, can make a difference in performance
jemalloc = ["dep:tikv-jemallocator-global", "cozorocks?/jemalloc"]
## Enables io-uring option for the RocksDB storage
io-uring = ["cozorocks?/io-uring"]
#! The following features are highly experimental:
## Enables the [Sled](https://github.com/spacejam/sled) backend
storage-sled = ["dep:sled"]
## Enables the [TiKV](https://tikv.org/) client backend
storage-tikv = ["dep:tikv-client", "dep:tokio"]
[dependencies] [dependencies]
casey = "0.3.3" casey = "0.3.3"
@ -51,23 +71,20 @@ itertools = "0.10.3"
regex = "1.6.0" regex = "1.6.0"
pest = "2.2.1" pest = "2.2.1"
pest_derive = "2.2.1" pest_derive = "2.2.1"
rayon = "1.5.3"
nalgebra = "0.31.1"
minreq = { version = "2.6.0", features = ["https-rustls"] }
approx = "0.5.1" approx = "0.5.1"
unicode-normalization = "0.1.21" unicode-normalization = "0.1.21"
thiserror = "1.0.34" thiserror = "1.0.34"
uuid = { version = "1.1.2", features = ["v1", "v4", "serde"] } uuid = { version = "1.1.2", features = ["v1", "v4", "serde"] }
csv = "1.1.6" csv = "1.1.6"
document-features = "0.2.6"
rayon = { version = "1.5.3", optional = true }
nalgebra = { version = "0.31.1", optional = true }
minreq = { version = "2.6.0", features = ["https-rustls"], optional = true }
tikv-jemallocator-global = { version = "0.5.0", optional = true } tikv-jemallocator-global = { version = "0.5.0", optional = true }
cozorocks = { path = "../cozorocks", version = "0.1.0" } cozorocks = { path = "../cozorocks", version = "0.1.0", optional = true }
sled = "0.34.7" sled = { version = "0.34.7", optional = true }
tikv-client = "0.1.0" tikv-client = { version = "0.1.0", optional = true }
tokio = "1.21.2" tokio = { version = "1.21.2", optional = true }
sqlite = "0.30.1" sqlite = { version = "0.30.1", optional = true }
#redb = "0.9.0" #redb = "0.9.0"
#ouroboros = "0.15.5" #ouroboros = "0.15.5"
clap = { version = "3.2.8", features = ["derive"] }
rouille = "3.5.0"

@ -9,6 +9,7 @@ use miette::{bail, ensure, IntoDiagnostic, Result};
use smartstring::{LazyCompact, SmartString}; use smartstring::{LazyCompact, SmartString};
use crate::algo::{AlgoImpl, CannotDetermineArity}; use crate::algo::{AlgoImpl, CannotDetermineArity};
#[cfg(feature = "requests")]
use crate::algo::jlines::get_file_content_from_url; use crate::algo::jlines::get_file_content_from_url;
use crate::data::expr::Expr; use crate::data::expr::Expr;
use crate::data::functions::{op_to_float, op_to_uuid}; use crate::data::functions::{op_to_float, op_to_uuid};
@ -158,12 +159,17 @@ impl AlgoImpl for CsvReader {
} }
} }
None => { None => {
let content = get_file_content_from_url(&url)?; #[cfg(feature = "requests")]
let mut rdr = rdr_builder.from_reader(content.as_bytes()); {
for record in rdr.records() { let content = get_file_content_from_url(&url)?;
let record = record.into_diagnostic()?; let mut rdr = rdr_builder.from_reader(content.as_bytes());
process_row(record)?; for record in rdr.records() {
let record = record.into_diagnostic()?;
process_row(record)?;
}
} }
#[cfg(not(feature = "requests"))]
bail!("the feature `requests` is not enabled for the build")
} }
} }
Ok(()) Ok(())

@ -9,7 +9,9 @@ use std::{fs, io};
use itertools::Itertools; use itertools::Itertools;
use log::error; use log::error;
#[allow(unused_imports)]
use miette::{bail, miette, Diagnostic, IntoDiagnostic, Result, WrapErr}; use miette::{bail, miette, Diagnostic, IntoDiagnostic, Result, WrapErr};
#[cfg(feature = "requests")]
use minreq::Response; use minreq::Response;
use smartstring::{LazyCompact, SmartString}; use smartstring::{LazyCompact, SmartString};
use thiserror::Error; use thiserror::Error;
@ -107,25 +109,30 @@ impl AlgoImpl for JsonReader {
} }
} }
None => { None => {
let content = get_file_content_from_url(&url)?; #[cfg(feature = "requests")]
let content = content.as_str().into_diagnostic()?; {
if json_lines { let content = get_file_content_from_url(&url)?;
for line in content.lines() { let content = content.as_str().into_diagnostic()?;
let line = line.trim(); if json_lines {
if !line.is_empty() { for line in content.lines() {
let row = serde_json::from_str(line).into_diagnostic()?; let line = line.trim();
process_row(&row)?; if !line.is_empty() {
let row = serde_json::from_str(line).into_diagnostic()?;
process_row(&row)?;
}
}
} else {
let data: JsonValue = serde_json::from_str(content).into_diagnostic()?;
let rows = data
.as_array()
.ok_or_else(|| miette!("JSON file is not an array"))?;
for row in rows {
process_row(row)?;
} }
}
} else {
let data: JsonValue = serde_json::from_str(content).into_diagnostic()?;
let rows = data
.as_array()
.ok_or_else(|| miette!("JSON file is not an array"))?;
for row in rows {
process_row(row)?;
} }
} }
#[cfg(not(feature = "requests"))]
bail!("the feature `requests` is not enabled for the build")
} }
} }
Ok(()) Ok(())
@ -171,6 +178,7 @@ impl AlgoImpl for JsonReader {
} }
} }
#[cfg(feature = "requests")]
pub(crate) fn get_file_content_from_url(url: &str) -> Result<Response> { pub(crate) fn get_file_content_from_url(url: &str) -> Result<Response> {
minreq::get(url as &str) minreq::get(url as &str)
.send() .send()

@ -8,25 +8,41 @@ use miette::{bail, ensure, Diagnostic, Result};
use smartstring::{LazyCompact, SmartString}; use smartstring::{LazyCompact, SmartString};
use thiserror::Error; use thiserror::Error;
#[cfg(feature = "graph-algo")]
use crate::algo::all_pairs_shortest_path::{BetweennessCentrality, ClosenessCentrality}; use crate::algo::all_pairs_shortest_path::{BetweennessCentrality, ClosenessCentrality};
#[cfg(feature = "graph-algo")]
use crate::algo::astar::ShortestPathAStar; use crate::algo::astar::ShortestPathAStar;
#[cfg(feature = "graph-algo")]
use crate::algo::bfs::Bfs; use crate::algo::bfs::Bfs;
use crate::algo::constant::Constant; use crate::algo::constant::Constant;
use crate::algo::csv::CsvReader; use crate::algo::csv::CsvReader;
#[cfg(feature = "graph-algo")]
use crate::algo::degree_centrality::DegreeCentrality; use crate::algo::degree_centrality::DegreeCentrality;
#[cfg(feature = "graph-algo")]
use crate::algo::dfs::Dfs; use crate::algo::dfs::Dfs;
use crate::algo::jlines::JsonReader; use crate::algo::jlines::JsonReader;
#[cfg(feature = "graph-algo")]
use crate::algo::kruskal::MinimumSpanningForestKruskal; use crate::algo::kruskal::MinimumSpanningForestKruskal;
#[cfg(feature = "graph-algo")]
use crate::algo::label_propagation::LabelPropagation; use crate::algo::label_propagation::LabelPropagation;
#[cfg(feature = "graph-algo")]
use crate::algo::louvain::CommunityDetectionLouvain; use crate::algo::louvain::CommunityDetectionLouvain;
#[cfg(feature = "graph-algo")]
use crate::algo::pagerank::PageRank; use crate::algo::pagerank::PageRank;
#[cfg(feature = "graph-algo")]
use crate::algo::prim::MinimumSpanningTreePrim; use crate::algo::prim::MinimumSpanningTreePrim;
#[cfg(feature = "graph-algo")]
use crate::algo::random_walk::RandomWalk; use crate::algo::random_walk::RandomWalk;
use crate::algo::reorder_sort::ReorderSort; use crate::algo::reorder_sort::ReorderSort;
#[cfg(feature = "graph-algo")]
use crate::algo::shortest_path_dijkstra::ShortestPathDijkstra; use crate::algo::shortest_path_dijkstra::ShortestPathDijkstra;
#[cfg(feature = "graph-algo")]
use crate::algo::strongly_connected_components::StronglyConnectedComponent; use crate::algo::strongly_connected_components::StronglyConnectedComponent;
#[cfg(feature = "graph-algo")]
use crate::algo::top_sort::TopSort; use crate::algo::top_sort::TopSort;
#[cfg(feature = "graph-algo")]
use crate::algo::triangles::ClusteringCoefficients; use crate::algo::triangles::ClusteringCoefficients;
#[cfg(feature = "graph-algo")]
use crate::algo::yen::KShortestPathYen; use crate::algo::yen::KShortestPathYen;
use crate::data::expr::Expr; use crate::data::expr::Expr;
use crate::data::program::{MagicAlgoApply, MagicAlgoRuleArg, MagicSymbol}; use crate::data::program::{MagicAlgoApply, MagicAlgoRuleArg, MagicSymbol};
@ -38,25 +54,40 @@ use crate::runtime::db::Poison;
use crate::runtime::in_mem::InMemRelation; use crate::runtime::in_mem::InMemRelation;
use crate::runtime::transact::SessionTx; use crate::runtime::transact::SessionTx;
#[cfg(feature = "graph-algo")]
pub(crate) mod all_pairs_shortest_path; pub(crate) mod all_pairs_shortest_path;
#[cfg(feature = "graph-algo")]
pub(crate) mod astar; pub(crate) mod astar;
#[cfg(feature = "graph-algo")]
pub(crate) mod bfs; pub(crate) mod bfs;
pub(crate) mod constant; pub(crate) mod constant;
pub(crate) mod csv; pub(crate) mod csv;
#[cfg(feature = "graph-algo")]
pub(crate) mod degree_centrality; pub(crate) mod degree_centrality;
#[cfg(feature = "graph-algo")]
pub(crate) mod dfs; pub(crate) mod dfs;
pub(crate) mod jlines; pub(crate) mod jlines;
#[cfg(feature = "graph-algo")]
pub(crate) mod kruskal; pub(crate) mod kruskal;
#[cfg(feature = "graph-algo")]
pub(crate) mod label_propagation; pub(crate) mod label_propagation;
#[cfg(feature = "graph-algo")]
pub(crate) mod louvain; pub(crate) mod louvain;
#[cfg(feature = "graph-algo")]
pub(crate) mod pagerank; pub(crate) mod pagerank;
#[cfg(feature = "graph-algo")]
pub(crate) mod prim; pub(crate) mod prim;
#[cfg(feature = "graph-algo")]
pub(crate) mod random_walk; pub(crate) mod random_walk;
pub(crate) mod reorder_sort; pub(crate) mod reorder_sort;
#[cfg(feature = "graph-algo")]
pub(crate) mod shortest_path_dijkstra; pub(crate) mod shortest_path_dijkstra;
pub(crate) mod strongly_connected_components; pub(crate) mod strongly_connected_components;
#[cfg(feature = "graph-algo")]
pub(crate) mod top_sort; pub(crate) mod top_sort;
#[cfg(feature = "graph-algo")]
pub(crate) mod triangles; pub(crate) mod triangles;
#[cfg(feature = "graph-algo")]
pub(crate) mod yen; pub(crate) mod yen;
pub(crate) trait AlgoImpl { pub(crate) trait AlgoImpl {
@ -106,25 +137,43 @@ impl AlgoHandle {
pub(crate) fn get_impl(&self) -> Result<Box<dyn AlgoImpl>> { pub(crate) fn get_impl(&self) -> Result<Box<dyn AlgoImpl>> {
Ok(match &self.name.name as &str { Ok(match &self.name.name as &str {
#[cfg(feature = "graph-algo")]
"ClusteringCoefficients" => Box::new(ClusteringCoefficients), "ClusteringCoefficients" => Box::new(ClusteringCoefficients),
#[cfg(feature = "graph-algo")]
"DegreeCentrality" => Box::new(DegreeCentrality), "DegreeCentrality" => Box::new(DegreeCentrality),
#[cfg(feature = "graph-algo")]
"ClosenessCentrality" => Box::new(ClosenessCentrality), "ClosenessCentrality" => Box::new(ClosenessCentrality),
#[cfg(feature = "graph-algo")]
"BetweennessCentrality" => Box::new(BetweennessCentrality), "BetweennessCentrality" => Box::new(BetweennessCentrality),
#[cfg(feature = "graph-algo")]
"DepthFirstSearch" | "DFS" => Box::new(Dfs), "DepthFirstSearch" | "DFS" => Box::new(Dfs),
#[cfg(feature = "graph-algo")]
"BreadthFirstSearch" | "BFS" => Box::new(Bfs), "BreadthFirstSearch" | "BFS" => Box::new(Bfs),
#[cfg(feature = "graph-algo")]
"ShortestPathDijkstra" => Box::new(ShortestPathDijkstra), "ShortestPathDijkstra" => Box::new(ShortestPathDijkstra),
#[cfg(feature = "graph-algo")]
"ShortestPathAStar" => Box::new(ShortestPathAStar), "ShortestPathAStar" => Box::new(ShortestPathAStar),
#[cfg(feature = "graph-algo")]
"KShortestPathYen" => Box::new(KShortestPathYen), "KShortestPathYen" => Box::new(KShortestPathYen),
#[cfg(feature = "graph-algo")]
"MinimumSpanningTreePrim" => Box::new(MinimumSpanningTreePrim), "MinimumSpanningTreePrim" => Box::new(MinimumSpanningTreePrim),
#[cfg(feature = "graph-algo")]
"MinimumSpanningForestKruskal" => Box::new(MinimumSpanningForestKruskal), "MinimumSpanningForestKruskal" => Box::new(MinimumSpanningForestKruskal),
#[cfg(feature = "graph-algo")]
"TopSort" => Box::new(TopSort), "TopSort" => Box::new(TopSort),
#[cfg(feature = "graph-algo")]
"ConnectedComponents" => Box::new(StronglyConnectedComponent::new(false)), "ConnectedComponents" => Box::new(StronglyConnectedComponent::new(false)),
#[cfg(feature = "graph-algo")]
"StronglyConnectedComponents" | "SCC" => { "StronglyConnectedComponents" | "SCC" => {
Box::new(StronglyConnectedComponent::new(true)) Box::new(StronglyConnectedComponent::new(true))
} }
#[cfg(feature = "graph-algo")]
"PageRank" => Box::new(PageRank), "PageRank" => Box::new(PageRank),
#[cfg(feature = "graph-algo")]
"CommunityDetectionLouvain" => Box::new(CommunityDetectionLouvain), "CommunityDetectionLouvain" => Box::new(CommunityDetectionLouvain),
#[cfg(feature = "graph-algo")]
"LabelPropagation" => Box::new(LabelPropagation), "LabelPropagation" => Box::new(LabelPropagation),
#[cfg(feature = "graph-algo")]
"RandomWalk" => Box::new(RandomWalk), "RandomWalk" => Box::new(RandomWalk),
"ReorderSort" => Box::new(ReorderSort), "ReorderSort" => Box::new(ReorderSort),
"JsonReader" => Box::new(JsonReader), "JsonReader" => Box::new(JsonReader),
@ -191,6 +240,7 @@ pub(crate) struct BadExprValueError(
pub(crate) struct AlgoNotFoundError(pub(crate) String, #[label] pub(crate) SourceSpan); pub(crate) struct AlgoNotFoundError(pub(crate) String, #[label] pub(crate) SourceSpan);
impl MagicAlgoRuleArg { impl MagicAlgoRuleArg {
#[allow(dead_code)]
pub(crate) fn convert_edge_to_weighted_graph<'a>( pub(crate) fn convert_edge_to_weighted_graph<'a>(
&'a self, &'a self,
undirected: bool, undirected: bool,
@ -276,6 +326,7 @@ impl MagicAlgoRuleArg {
} }
Ok((graph, indices, inv_indices, has_neg_edge)) Ok((graph, indices, inv_indices, has_neg_edge))
} }
#[allow(dead_code)]
pub(crate) fn convert_edge_to_graph<'a>( pub(crate) fn convert_edge_to_graph<'a>(
&'a self, &'a self,
undirected: bool, undirected: bool,
@ -315,7 +366,7 @@ impl MagicAlgoRuleArg {
} }
Ok((graph, indices, inv_indices)) Ok((graph, indices, inv_indices))
} }
#[allow(dead_code)]
pub(crate) fn prefix_iter<'a>( pub(crate) fn prefix_iter<'a>(
&'a self, &'a self,
prefix: &DataValue, prefix: &DataValue,

@ -1,6 +1,7 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under MPL-2.0. * Copyright 2022, The Cozo Project Authors. Licensed under MPL-2.0.
*/ */
#![allow(unused_imports)]
use std::cmp::min; use std::cmp::min;
use std::collections::BTreeMap; use std::collections::BTreeMap;
@ -20,16 +21,18 @@ use crate::runtime::db::Poison;
use crate::runtime::in_mem::InMemRelation; use crate::runtime::in_mem::InMemRelation;
use crate::runtime::transact::SessionTx; use crate::runtime::transact::SessionTx;
#[cfg(feature = "graph-algo")]
pub(crate) struct StronglyConnectedComponent { pub(crate) struct StronglyConnectedComponent {
strong: bool, strong: bool,
} }
#[cfg(feature = "graph-algo")]
impl StronglyConnectedComponent { impl StronglyConnectedComponent {
pub(crate) fn new(strong: bool) -> Self { pub(crate) fn new(strong: bool) -> Self {
Self { strong } Self { strong }
} }
} }
#[cfg(feature = "graph-algo")]
impl AlgoImpl for StronglyConnectedComponent { impl AlgoImpl for StronglyConnectedComponent {
fn run<'a>( fn run<'a>(
&mut self, &mut self,

@ -195,11 +195,13 @@ impl Expr {
} }
Ok(()) Ok(())
} }
#[allow(dead_code)]
pub(crate) fn binding_indices(&self) -> BTreeSet<usize> { pub(crate) fn binding_indices(&self) -> BTreeSet<usize> {
let mut ret = BTreeSet::default(); let mut ret = BTreeSet::default();
self.do_binding_indices(&mut ret); self.do_binding_indices(&mut ret);
ret ret
} }
#[allow(dead_code)]
fn do_binding_indices(&self, coll: &mut BTreeSet<usize>) { fn do_binding_indices(&self, coll: &mut BTreeSet<usize>) {
match self { match self {
Expr::Binding { tuple_pos, .. } => { Expr::Binding { tuple_pos, .. } => {

@ -1441,7 +1441,10 @@ pub(crate) fn op_format_timestamp(args: &[DataValue]) -> Result<DataValue> {
.get_float() .get_float()
.ok_or_else(|| miette!("'format_timestamp' expects a number"))?; .ok_or_else(|| miette!("'format_timestamp' expects a number"))?;
let millis = (f * 1000.) as i64; let millis = (f * 1000.) as i64;
let dt = Utc.timestamp_millis(millis); let dt = Utc
.timestamp_millis_opt(millis)
.latest()
.ok_or_else(|| miette!("bad time: {}", f))?;
match args.get(1) { match args.get(1) {
Some(tz_v) => { Some(tz_v) => {
let tz_s = tz_v.get_string().ok_or_else(|| { let tz_s = tz_v.get_string().ok_or_else(|| {

@ -267,6 +267,7 @@ pub(crate) struct WrongAlgoOptionError {
} }
impl MagicAlgoApply { impl MagicAlgoApply {
#[allow(dead_code)]
pub(crate) fn relation_with_min_len( pub(crate) fn relation_with_min_len(
&self, &self,
idx: usize, idx: usize,
@ -349,6 +350,7 @@ impl MagicAlgoApply {
}, },
} }
} }
#[allow(dead_code)]
pub(crate) fn pos_integer_option(&self, name: &str, default: Option<usize>) -> Result<usize> { pub(crate) fn pos_integer_option(&self, name: &str, default: Option<usize>) -> Result<usize> {
match self.options.get(name) { match self.options.get(name) {
Some(v) => match v.clone().eval_to_const() { Some(v) => match v.clone().eval_to_const() {
@ -437,6 +439,7 @@ impl MagicAlgoApply {
}, },
} }
} }
#[allow(dead_code)]
pub(crate) fn unit_interval_option(&self, name: &str, default: Option<f64>) -> Result<f64> { pub(crate) fn unit_interval_option(&self, name: &str, default: Option<f64>) -> Result<f64> {
match self.options.get(name) { match self.options.get(name) {
Some(v) => match v.clone().eval_to_const() { Some(v) => match v.clone().eval_to_const() {
@ -571,12 +574,14 @@ pub(crate) enum MagicAlgoRuleArg {
} }
impl MagicAlgoRuleArg { impl MagicAlgoRuleArg {
#[allow(dead_code)]
pub(crate) fn bindings(&self) -> &[Symbol] { pub(crate) fn bindings(&self) -> &[Symbol] {
match self { match self {
MagicAlgoRuleArg::InMem { bindings, .. } MagicAlgoRuleArg::InMem { bindings, .. }
| MagicAlgoRuleArg::Stored { bindings, .. } => bindings, | MagicAlgoRuleArg::Stored { bindings, .. } => bindings,
} }
} }
#[allow(dead_code)]
pub(crate) fn span(&self) -> SourceSpan { pub(crate) fn span(&self) -> SourceSpan {
match self { match self {
MagicAlgoRuleArg::InMem { span, .. } | MagicAlgoRuleArg::Stored { span, .. } => *span, MagicAlgoRuleArg::InMem { span, .. } | MagicAlgoRuleArg::Stored { span, .. } => *span,

@ -10,8 +10,6 @@ use crate::data::memcmp::MemCmpEncoder;
use crate::data::value::DataValue; use crate::data::value::DataValue;
use crate::runtime::relation::RelationId; use crate::runtime::relation::RelationId;
pub(crate) const KEY_PREFIX_LEN: usize = 9;
#[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Default)] #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Default)]
pub struct Tuple(pub(crate) Vec<DataValue>); pub struct Tuple(pub(crate) Vec<DataValue>);

@ -23,7 +23,7 @@
//! We created an in-memory database with [`new_cozo_mem`](crate::new_cozo_mem) above. //! We created an in-memory database with [`new_cozo_mem`](crate::new_cozo_mem) above.
//! Persistent options include [`new_cozo_rocksdb`](crate::new_cozo_rocksdb), //! Persistent options include [`new_cozo_rocksdb`](crate::new_cozo_rocksdb),
//! [`new_cozo_sqlite`](crate::new_cozo_sqlite) and others. //! [`new_cozo_sqlite`](crate::new_cozo_sqlite) and others.
#![doc = document_features::document_features!()]
#![warn(rust_2018_idioms, future_incompatible)] #![warn(rust_2018_idioms, future_incompatible)]
#![warn(missing_docs)] #![warn(missing_docs)]
#![allow(clippy::type_complexity)] #![allow(clippy::type_complexity)]
@ -34,9 +34,13 @@ pub use miette::Error;
pub use runtime::db::Db; pub use runtime::db::Db;
pub use runtime::relation::decode_tuple_from_kv; pub use runtime::relation::decode_tuple_from_kv;
pub use storage::mem::{new_cozo_mem, MemStorage}; pub use storage::mem::{new_cozo_mem, MemStorage};
#[cfg(feature = "storage-rocksdb")]
pub use storage::rocks::{new_cozo_rocksdb, RocksDbStorage}; pub use storage::rocks::{new_cozo_rocksdb, RocksDbStorage};
#[cfg(feature = "storage-sled")]
pub use storage::sled::{new_cozo_sled, SledStorage}; pub use storage::sled::{new_cozo_sled, SledStorage};
#[cfg(feature = "storage-sqlite")]
pub use storage::sqlite::{new_cozo_sqlite, SqliteStorage}; pub use storage::sqlite::{new_cozo_sqlite, SqliteStorage};
#[cfg(feature = "storage-tikv")]
pub use storage::tikv::{new_cozo_tikv, TiKvStorage}; pub use storage::tikv::{new_cozo_tikv, TiKvStorage};
pub use storage::{Storage, StoreTx}; pub use storage::{Storage, StoreTx};

@ -59,9 +59,6 @@ pub struct DbManifest {
pub storage_version: u64, pub storage_version: u64,
} }
// FIXME this should be storage-specific
pub(crate) const CURRENT_STORAGE_VERSION: u64 = 1;
/// The database object of Cozo. /// The database object of Cozo.
#[derive(Clone)] #[derive(Clone)]
pub struct Db<S> { pub struct Db<S> {

@ -9,9 +9,13 @@ use crate::data::tuple::Tuple;
use crate::decode_tuple_from_kv; use crate::decode_tuple_from_kv;
pub(crate) mod mem; pub(crate) mod mem;
#[cfg(feature = "storage-rocksdb")]
pub(crate) mod rocks; pub(crate) mod rocks;
#[cfg(feature = "storage-sled")]
pub(crate) mod sled; pub(crate) mod sled;
#[cfg(feature = "storage-sqlite")]
pub(crate) mod sqlite; pub(crate) mod sqlite;
#[cfg(feature = "storage-tikv")]
pub(crate) mod tikv; pub(crate) mod tikv;
// pub(crate) mod re; // pub(crate) mod re;
@ -58,7 +62,7 @@ pub trait StoreTx<'s> {
fn commit(&mut self) -> Result<()>; fn commit(&mut self) -> Result<()>;
/// Scan on a range. `lower` is inclusive whereas `upper` is exclusive. /// Scan on a range. `lower` is inclusive whereas `upper` is exclusive.
/// The default implementation calls [`range_scan_owned`](Self::range_scan_owned) and converts the results. /// The default implementation calls [`range_scan_owned`](Self::range_scan) and converts the results.
/// ///
/// The implementation must call [`decode_tuple_from_kv`](crate::decode_tuple_from_kv) to obtain /// The implementation must call [`decode_tuple_from_kv`](crate::decode_tuple_from_kv) to obtain
/// a decoded tuple in the loop of the iterator. /// a decoded tuple in the loop of the iterator.

@ -9,13 +9,16 @@ use miette::{miette, IntoDiagnostic, Result, WrapErr};
use cozorocks::{DbBuilder, DbIter, RocksDb, Tx}; use cozorocks::{DbBuilder, DbIter, RocksDb, Tx};
use crate::data::tuple::{Tuple, KEY_PREFIX_LEN}; use crate::data::tuple::Tuple;
use crate::runtime::db::{BadDbInit, DbManifest, CURRENT_STORAGE_VERSION}; use crate::runtime::db::{BadDbInit, DbManifest};
use crate::runtime::relation::decode_tuple_from_kv; use crate::runtime::relation::decode_tuple_from_kv;
use crate::storage::{Storage, StoreTx}; use crate::storage::{Storage, StoreTx};
use crate::utils::swap_option_result; use crate::utils::swap_option_result;
use crate::Db; use crate::Db;
const KEY_PREFIX_LEN: usize = 9;
const CURRENT_STORAGE_VERSION: u64 = 1;
/// Creates a RocksDB database object. /// Creates a RocksDB database object.
/// This is currently the fastest persistent storage and it can /// This is currently the fastest persistent storage and it can
/// sustain huge concurrency. /// sustain huge concurrency.

@ -1,6 +1,7 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later.
*/ */
#![cfg(feature = "graph-algo")]
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant; use std::time::Instant;

@ -1,6 +1,8 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later.
*/ */
#![cfg(feature = "storage-rocksdb")]
#![cfg(feature = "graph-algo")]
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant; use std::time::Instant;

@ -1,6 +1,8 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later.
*/ */
#![cfg(feature = "storage-sled")]
#![cfg(feature = "graph-algo")]
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant; use std::time::Instant;

@ -1,6 +1,8 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later.
*/ */
#![cfg(feature = "storage-sqlite")]
#![cfg(feature = "graph-algo")]
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant; use std::time::Instant;

@ -1,16 +1,19 @@
/* /*
* Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later.
*/ */
#![cfg(feature = "storage-tikv")]
#![cfg(feature = "graph-algo")]
use std::str::FromStr; use std::str::FromStr;
use std::time::Instant; use std::time::Instant;
use approx::AbsDiffEq; use approx::AbsDiffEq;
use cozo::{new_cozo_tikv, Db, TiKvStorage};
use env_logger::Env; use env_logger::Env;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use serde_json::json; use serde_json::json;
use cozo::{new_cozo_tikv, Db, TiKvStorage};
lazy_static! { lazy_static! {
static ref TEST_DB: Db<TiKvStorage> = { static ref TEST_DB: Db<TiKvStorage> = {
env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();

@ -0,0 +1,11 @@
[package]
name = "cozoserver"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
cozo = { version = "0.1.2", path = "../cozo-core" }
clap = { version = "3.2.8", features = ["derive"] }
rouille = "3.5.0"
Loading…
Cancel
Save