From a863dd0de1e7a4b88f691c3232e5195755e1fdb3 Mon Sep 17 00:00:00 2001 From: Ziyang Hu Date: Mon, 14 Nov 2022 00:06:28 +0800 Subject: [PATCH] move stuff around; document features --- Cargo.lock | 113 +++++++++++------- Cargo.toml | 2 +- cozo-core/Cargo.toml | 53 +++++--- cozo-core/src/algo/csv.rs | 16 ++- cozo-core/src/algo/jlines.rs | 40 ++++--- cozo-core/src/algo/mod.rs | 53 +++++++- .../src/algo/strongly_connected_components.rs | 5 +- cozo-core/src/data/expr.rs | 2 + cozo-core/src/data/functions.rs | 5 +- cozo-core/src/data/program.rs | 5 + cozo-core/src/data/tuple.rs | 2 - cozo-core/src/lib.rs | 6 +- cozo-core/src/runtime/db.rs | 3 - cozo-core/src/storage/mod.rs | 6 +- cozo-core/src/storage/rocks.rs | 7 +- cozo-core/tests/air_routes_mem.rs | 1 + cozo-core/tests/air_routes_rocksdb.rs | 2 + cozo-core/tests/air_routes_sled.rs | 2 + cozo-core/tests/air_routes_sqlite.rs | 2 + cozo-core/tests/air_routes_tikv.rs | 5 +- cozoserver/Cargo.toml | 11 ++ .../cozoserver.rs => cozoserver/src/main.rs | 0 22 files changed, 243 insertions(+), 98 deletions(-) create mode 100644 cozoserver/Cargo.toml rename cozo-core/src/bin/cozoserver.rs => cozoserver/src/main.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index c85e67a6..51a181a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,9 +250,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytemuck" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aec14f5d4e6e3f927cd0c81f72e5710d95ee9019fbeb4b3021193867491bfd8" +checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f" [[package]] name = "byteorder" @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.74" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" +checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" dependencies = [ "jobserver", ] @@ -332,9 +332,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", "js-sys", @@ -478,9 +478,9 @@ dependencies = [ "casey", "chrono", "chrono-tz", - "clap", "cozorocks", "csv", + "document-features", "either", "env_logger", "itertools 0.10.5", @@ -500,7 +500,6 @@ dependencies = [ "rmp", "rmp-serde", "rmpv", - "rouille", "serde", "serde_bytes", "serde_derive", @@ -550,6 +549,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "cozoserver" +version = "0.1.0" +dependencies = [ + "clap", + "cozo", + "rouille", +] + [[package]] name = "cpufeatures" version = "0.2.5" @@ -645,9 +653,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +checksum = "97abf9f0eca9e52b7f81b945524e76710e6cb2366aead23b7d4fbf72e281f888" dependencies = [ "cc", "cxxbridge-flags", @@ -657,9 +665,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +checksum = "7cc32cc5fea1d894b77d269ddb9f192110069a8a9c1f1d441195fba90553dea3" dependencies = [ "cc", "codespan-reporting", @@ -672,15 +680,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" +checksum = "8ca220e4794c934dc6b1207c3b42856ad4c302f2df1712e9f8d2eec5afaacf1f" [[package]] name = "cxxbridge-macro" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +checksum = "b846f081361125bfc8dc9d3940c84e1fd83ba54bbca7b17cd29483c828be0704" dependencies = [ "proc-macro2", "quote", @@ -753,6 +761,15 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "document-features" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3267e1ade4f1f6ddd35fed44a04b6514e244ffeda90c6a14a9ee30f9c9fd7a1" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.8.0" @@ -770,9 +787,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ "atty", "humantime", @@ -1424,6 +1441,12 @@ dependencies = [ "cc", ] +[[package]] +name = "litrs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9275e0933cf8bb20f008924c0cb07a0692fe54d8064996520bf998de9eb79aa" + [[package]] name = "lock_api" version = "0.4.9" @@ -1678,9 +1701,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", @@ -1767,18 +1790,18 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f74e330193f90ec45e2b257fa3ef6df087784157ac1ad2c1e71c62837b03aa7" +checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf" dependencies = [ "num-traits", ] [[package]] name = "os_str_bytes" -version = "6.3.1" +version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e" [[package]] name = "owo-colors" @@ -1840,9 +1863,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" +checksum = "a528564cc62c19a7acac4d81e01f39e53e25e17b934878f4c6d25cc2836e62f8" dependencies = [ "thiserror", "ucd-trie", @@ -1850,9 +1873,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2" +checksum = "d5fd9bc6500181952d34bd0b2b0163a54d794227b498be0b7afa7698d0a7b18f" dependencies = [ "pest", "pest_generator", @@ -1860,9 +1883,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db" +checksum = "d2610d5ac5156217b4ff8e46ddcef7cdf44b273da2ac5bca2ecbfa86a330e7c4" dependencies = [ "pest", "pest_meta", @@ -1873,9 +1896,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d" +checksum = "824749bf7e21dd66b36fbe26b3f45c713879cccd4a009a917ab8e045ca8246fe" dependencies = [ "once_cell", "pest", @@ -1961,9 +1984,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" @@ -1977,9 +2000,9 @@ dependencies = [ [[package]] name = "priority-queue" -version = "1.2.3" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "815082d99af3acc75a3e67efd2a07f72e67b4e81b4344eb8ca34c6ebf3dfa9c5" +checksum = "d7685ca4cc0b3ad748c22ce6803e23b55b9206ef7715b965ebeaf41639238fdc" dependencies = [ "autocfg", "indexmap", @@ -2303,9 +2326,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -2320,9 +2343,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -2464,7 +2487,7 @@ dependencies = [ "serde_json", "sha1", "threadpool", - "time 0.3.16", + "time 0.3.17", "tiny_http", "url", ] @@ -2784,9 +2807,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "supports-color" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4872ced36b91d47bae8a214a683fe54e7078875b399dfa251df346c9b547d1f9" +checksum = "8ba6faf2ca7ee42fdd458f4347ae0a9bd6bcc445ad7cb57ad82b383f18870d6f" dependencies = [ "atty", "is_ci", @@ -3030,9 +3053,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "libc", "num_threads", diff --git a/Cargo.toml b/Cargo.toml index a6d6d367..b915c6cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,4 +2,4 @@ lto = true [workspace] -members = ["cozorocks", "cozo-lib-c", "cozo-lib-java", "cozo-core"] +members = ["cozorocks", "cozo-lib-c", "cozo-lib-java", "cozo-core", "cozoserver"] diff --git a/cozo-core/Cargo.toml b/cozo-core/Cargo.toml index 68ca7188..a75cb6c8 100644 --- a/cozo-core/Cargo.toml +++ b/cozo-core/Cargo.toml @@ -16,12 +16,32 @@ exclude = [ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -storage-rocksdb = [] -storage-sled = [] -storage-tikv = [] -#storage-redb = [] -jemalloc = ["tikv-jemallocator-global", "cozorocks/jemalloc"] -io-uring = ["cozorocks/io-uring"] +#! # Features + +default = ["compact", "storage-rocksdb"] +## Enables the `minimal`, `requests` and `graph-algo` features +compact = ["minimal", "requests", "graph-algo"] +## Enables the `storage-sqlite` feature +minimal = ["storage-sqlite"] +## Enables the [Sqlite](https://www.sqlite.org/index.html) backend, also allows backup and restore with Sqlite data files. +storage-sqlite = ["dep:sqlite"] +## Enables the [RocksDB](http://rocksdb.org/) backend +storage-rocksdb = ["dep:cozorocks"] +## Enables the graph algorithms +graph-algo = ["dep:rayon", "dep:nalgebra"] +## Allows the utilities to make web requests to fetch data +requests = ["dep:minreq"] +## Uses jemalloc as the global allocator, can make a difference in performance +jemalloc = ["dep:tikv-jemallocator-global", "cozorocks?/jemalloc"] +## Enables io-uring option for the RocksDB storage +io-uring = ["cozorocks?/io-uring"] + +#! The following features are highly experimental: + +## Enables the [Sled](https://github.com/spacejam/sled) backend +storage-sled = ["dep:sled"] +## Enables the [TiKV](https://tikv.org/) client backend +storage-tikv = ["dep:tikv-client", "dep:tokio"] [dependencies] casey = "0.3.3" @@ -51,23 +71,20 @@ itertools = "0.10.3" regex = "1.6.0" pest = "2.2.1" pest_derive = "2.2.1" -rayon = "1.5.3" -nalgebra = "0.31.1" -minreq = { version = "2.6.0", features = ["https-rustls"] } approx = "0.5.1" unicode-normalization = "0.1.21" thiserror = "1.0.34" uuid = { version = "1.1.2", features = ["v1", "v4", "serde"] } csv = "1.1.6" +document-features = "0.2.6" +rayon = { version = "1.5.3", optional = true } +nalgebra = { version = "0.31.1", optional = true } +minreq = { version = "2.6.0", features = ["https-rustls"], optional = true } tikv-jemallocator-global = { version = "0.5.0", optional = true } -cozorocks = { path = "../cozorocks", version = "0.1.0" } -sled = "0.34.7" -tikv-client = "0.1.0" -tokio = "1.21.2" -sqlite = "0.30.1" +cozorocks = { path = "../cozorocks", version = "0.1.0", optional = true } +sled = { version = "0.34.7", optional = true } +tikv-client = { version = "0.1.0", optional = true } +tokio = { version = "1.21.2", optional = true } +sqlite = { version = "0.30.1", optional = true } #redb = "0.9.0" #ouroboros = "0.15.5" - -clap = { version = "3.2.8", features = ["derive"] } -rouille = "3.5.0" - diff --git a/cozo-core/src/algo/csv.rs b/cozo-core/src/algo/csv.rs index 7ebf3aa9..de72cfcf 100644 --- a/cozo-core/src/algo/csv.rs +++ b/cozo-core/src/algo/csv.rs @@ -9,6 +9,7 @@ use miette::{bail, ensure, IntoDiagnostic, Result}; use smartstring::{LazyCompact, SmartString}; use crate::algo::{AlgoImpl, CannotDetermineArity}; +#[cfg(feature = "requests")] use crate::algo::jlines::get_file_content_from_url; use crate::data::expr::Expr; use crate::data::functions::{op_to_float, op_to_uuid}; @@ -158,12 +159,17 @@ impl AlgoImpl for CsvReader { } } None => { - let content = get_file_content_from_url(&url)?; - let mut rdr = rdr_builder.from_reader(content.as_bytes()); - for record in rdr.records() { - let record = record.into_diagnostic()?; - process_row(record)?; + #[cfg(feature = "requests")] + { + let content = get_file_content_from_url(&url)?; + let mut rdr = rdr_builder.from_reader(content.as_bytes()); + for record in rdr.records() { + let record = record.into_diagnostic()?; + process_row(record)?; + } } + #[cfg(not(feature = "requests"))] + bail!("the feature `requests` is not enabled for the build") } } Ok(()) diff --git a/cozo-core/src/algo/jlines.rs b/cozo-core/src/algo/jlines.rs index 3788323c..877db54c 100644 --- a/cozo-core/src/algo/jlines.rs +++ b/cozo-core/src/algo/jlines.rs @@ -9,7 +9,9 @@ use std::{fs, io}; use itertools::Itertools; use log::error; +#[allow(unused_imports)] use miette::{bail, miette, Diagnostic, IntoDiagnostic, Result, WrapErr}; +#[cfg(feature = "requests")] use minreq::Response; use smartstring::{LazyCompact, SmartString}; use thiserror::Error; @@ -107,25 +109,30 @@ impl AlgoImpl for JsonReader { } } None => { - let content = get_file_content_from_url(&url)?; - let content = content.as_str().into_diagnostic()?; - if json_lines { - for line in content.lines() { - let line = line.trim(); - if !line.is_empty() { - let row = serde_json::from_str(line).into_diagnostic()?; - process_row(&row)?; + #[cfg(feature = "requests")] + { + let content = get_file_content_from_url(&url)?; + let content = content.as_str().into_diagnostic()?; + if json_lines { + for line in content.lines() { + let line = line.trim(); + if !line.is_empty() { + let row = serde_json::from_str(line).into_diagnostic()?; + process_row(&row)?; + } + } + } else { + let data: JsonValue = serde_json::from_str(content).into_diagnostic()?; + let rows = data + .as_array() + .ok_or_else(|| miette!("JSON file is not an array"))?; + for row in rows { + process_row(row)?; } - } - } else { - let data: JsonValue = serde_json::from_str(content).into_diagnostic()?; - let rows = data - .as_array() - .ok_or_else(|| miette!("JSON file is not an array"))?; - for row in rows { - process_row(row)?; } } + #[cfg(not(feature = "requests"))] + bail!("the feature `requests` is not enabled for the build") } } Ok(()) @@ -171,6 +178,7 @@ impl AlgoImpl for JsonReader { } } +#[cfg(feature = "requests")] pub(crate) fn get_file_content_from_url(url: &str) -> Result { minreq::get(url as &str) .send() diff --git a/cozo-core/src/algo/mod.rs b/cozo-core/src/algo/mod.rs index 806facda..059fc464 100644 --- a/cozo-core/src/algo/mod.rs +++ b/cozo-core/src/algo/mod.rs @@ -8,25 +8,41 @@ use miette::{bail, ensure, Diagnostic, Result}; use smartstring::{LazyCompact, SmartString}; use thiserror::Error; +#[cfg(feature = "graph-algo")] use crate::algo::all_pairs_shortest_path::{BetweennessCentrality, ClosenessCentrality}; +#[cfg(feature = "graph-algo")] use crate::algo::astar::ShortestPathAStar; +#[cfg(feature = "graph-algo")] use crate::algo::bfs::Bfs; use crate::algo::constant::Constant; use crate::algo::csv::CsvReader; +#[cfg(feature = "graph-algo")] use crate::algo::degree_centrality::DegreeCentrality; +#[cfg(feature = "graph-algo")] use crate::algo::dfs::Dfs; use crate::algo::jlines::JsonReader; +#[cfg(feature = "graph-algo")] use crate::algo::kruskal::MinimumSpanningForestKruskal; +#[cfg(feature = "graph-algo")] use crate::algo::label_propagation::LabelPropagation; +#[cfg(feature = "graph-algo")] use crate::algo::louvain::CommunityDetectionLouvain; +#[cfg(feature = "graph-algo")] use crate::algo::pagerank::PageRank; +#[cfg(feature = "graph-algo")] use crate::algo::prim::MinimumSpanningTreePrim; +#[cfg(feature = "graph-algo")] use crate::algo::random_walk::RandomWalk; use crate::algo::reorder_sort::ReorderSort; +#[cfg(feature = "graph-algo")] use crate::algo::shortest_path_dijkstra::ShortestPathDijkstra; +#[cfg(feature = "graph-algo")] use crate::algo::strongly_connected_components::StronglyConnectedComponent; +#[cfg(feature = "graph-algo")] use crate::algo::top_sort::TopSort; +#[cfg(feature = "graph-algo")] use crate::algo::triangles::ClusteringCoefficients; +#[cfg(feature = "graph-algo")] use crate::algo::yen::KShortestPathYen; use crate::data::expr::Expr; use crate::data::program::{MagicAlgoApply, MagicAlgoRuleArg, MagicSymbol}; @@ -38,25 +54,40 @@ use crate::runtime::db::Poison; use crate::runtime::in_mem::InMemRelation; use crate::runtime::transact::SessionTx; +#[cfg(feature = "graph-algo")] pub(crate) mod all_pairs_shortest_path; +#[cfg(feature = "graph-algo")] pub(crate) mod astar; +#[cfg(feature = "graph-algo")] pub(crate) mod bfs; pub(crate) mod constant; pub(crate) mod csv; +#[cfg(feature = "graph-algo")] pub(crate) mod degree_centrality; +#[cfg(feature = "graph-algo")] pub(crate) mod dfs; pub(crate) mod jlines; +#[cfg(feature = "graph-algo")] pub(crate) mod kruskal; +#[cfg(feature = "graph-algo")] pub(crate) mod label_propagation; +#[cfg(feature = "graph-algo")] pub(crate) mod louvain; +#[cfg(feature = "graph-algo")] pub(crate) mod pagerank; +#[cfg(feature = "graph-algo")] pub(crate) mod prim; +#[cfg(feature = "graph-algo")] pub(crate) mod random_walk; pub(crate) mod reorder_sort; +#[cfg(feature = "graph-algo")] pub(crate) mod shortest_path_dijkstra; pub(crate) mod strongly_connected_components; +#[cfg(feature = "graph-algo")] pub(crate) mod top_sort; +#[cfg(feature = "graph-algo")] pub(crate) mod triangles; +#[cfg(feature = "graph-algo")] pub(crate) mod yen; pub(crate) trait AlgoImpl { @@ -106,25 +137,43 @@ impl AlgoHandle { pub(crate) fn get_impl(&self) -> Result> { Ok(match &self.name.name as &str { + #[cfg(feature = "graph-algo")] "ClusteringCoefficients" => Box::new(ClusteringCoefficients), + #[cfg(feature = "graph-algo")] "DegreeCentrality" => Box::new(DegreeCentrality), + #[cfg(feature = "graph-algo")] "ClosenessCentrality" => Box::new(ClosenessCentrality), + #[cfg(feature = "graph-algo")] "BetweennessCentrality" => Box::new(BetweennessCentrality), + #[cfg(feature = "graph-algo")] "DepthFirstSearch" | "DFS" => Box::new(Dfs), + #[cfg(feature = "graph-algo")] "BreadthFirstSearch" | "BFS" => Box::new(Bfs), + #[cfg(feature = "graph-algo")] "ShortestPathDijkstra" => Box::new(ShortestPathDijkstra), + #[cfg(feature = "graph-algo")] "ShortestPathAStar" => Box::new(ShortestPathAStar), + #[cfg(feature = "graph-algo")] "KShortestPathYen" => Box::new(KShortestPathYen), + #[cfg(feature = "graph-algo")] "MinimumSpanningTreePrim" => Box::new(MinimumSpanningTreePrim), + #[cfg(feature = "graph-algo")] "MinimumSpanningForestKruskal" => Box::new(MinimumSpanningForestKruskal), + #[cfg(feature = "graph-algo")] "TopSort" => Box::new(TopSort), + #[cfg(feature = "graph-algo")] "ConnectedComponents" => Box::new(StronglyConnectedComponent::new(false)), + #[cfg(feature = "graph-algo")] "StronglyConnectedComponents" | "SCC" => { Box::new(StronglyConnectedComponent::new(true)) } + #[cfg(feature = "graph-algo")] "PageRank" => Box::new(PageRank), + #[cfg(feature = "graph-algo")] "CommunityDetectionLouvain" => Box::new(CommunityDetectionLouvain), + #[cfg(feature = "graph-algo")] "LabelPropagation" => Box::new(LabelPropagation), + #[cfg(feature = "graph-algo")] "RandomWalk" => Box::new(RandomWalk), "ReorderSort" => Box::new(ReorderSort), "JsonReader" => Box::new(JsonReader), @@ -191,6 +240,7 @@ pub(crate) struct BadExprValueError( pub(crate) struct AlgoNotFoundError(pub(crate) String, #[label] pub(crate) SourceSpan); impl MagicAlgoRuleArg { + #[allow(dead_code)] pub(crate) fn convert_edge_to_weighted_graph<'a>( &'a self, undirected: bool, @@ -276,6 +326,7 @@ impl MagicAlgoRuleArg { } Ok((graph, indices, inv_indices, has_neg_edge)) } + #[allow(dead_code)] pub(crate) fn convert_edge_to_graph<'a>( &'a self, undirected: bool, @@ -315,7 +366,7 @@ impl MagicAlgoRuleArg { } Ok((graph, indices, inv_indices)) } - + #[allow(dead_code)] pub(crate) fn prefix_iter<'a>( &'a self, prefix: &DataValue, diff --git a/cozo-core/src/algo/strongly_connected_components.rs b/cozo-core/src/algo/strongly_connected_components.rs index 551c9a1e..08bbea4b 100644 --- a/cozo-core/src/algo/strongly_connected_components.rs +++ b/cozo-core/src/algo/strongly_connected_components.rs @@ -1,6 +1,7 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under MPL-2.0. */ +#![allow(unused_imports)] use std::cmp::min; use std::collections::BTreeMap; @@ -20,16 +21,18 @@ use crate::runtime::db::Poison; use crate::runtime::in_mem::InMemRelation; use crate::runtime::transact::SessionTx; +#[cfg(feature = "graph-algo")] pub(crate) struct StronglyConnectedComponent { strong: bool, } - +#[cfg(feature = "graph-algo")] impl StronglyConnectedComponent { pub(crate) fn new(strong: bool) -> Self { Self { strong } } } +#[cfg(feature = "graph-algo")] impl AlgoImpl for StronglyConnectedComponent { fn run<'a>( &mut self, diff --git a/cozo-core/src/data/expr.rs b/cozo-core/src/data/expr.rs index cd65672c..3d3a4fb0 100644 --- a/cozo-core/src/data/expr.rs +++ b/cozo-core/src/data/expr.rs @@ -195,11 +195,13 @@ impl Expr { } Ok(()) } + #[allow(dead_code)] pub(crate) fn binding_indices(&self) -> BTreeSet { let mut ret = BTreeSet::default(); self.do_binding_indices(&mut ret); ret } + #[allow(dead_code)] fn do_binding_indices(&self, coll: &mut BTreeSet) { match self { Expr::Binding { tuple_pos, .. } => { diff --git a/cozo-core/src/data/functions.rs b/cozo-core/src/data/functions.rs index b6632560..73d6b1ed 100644 --- a/cozo-core/src/data/functions.rs +++ b/cozo-core/src/data/functions.rs @@ -1441,7 +1441,10 @@ pub(crate) fn op_format_timestamp(args: &[DataValue]) -> Result { .get_float() .ok_or_else(|| miette!("'format_timestamp' expects a number"))?; let millis = (f * 1000.) as i64; - let dt = Utc.timestamp_millis(millis); + let dt = Utc + .timestamp_millis_opt(millis) + .latest() + .ok_or_else(|| miette!("bad time: {}", f))?; match args.get(1) { Some(tz_v) => { let tz_s = tz_v.get_string().ok_or_else(|| { diff --git a/cozo-core/src/data/program.rs b/cozo-core/src/data/program.rs index 1a72605b..3053deb6 100644 --- a/cozo-core/src/data/program.rs +++ b/cozo-core/src/data/program.rs @@ -267,6 +267,7 @@ pub(crate) struct WrongAlgoOptionError { } impl MagicAlgoApply { + #[allow(dead_code)] pub(crate) fn relation_with_min_len( &self, idx: usize, @@ -349,6 +350,7 @@ impl MagicAlgoApply { }, } } + #[allow(dead_code)] pub(crate) fn pos_integer_option(&self, name: &str, default: Option) -> Result { match self.options.get(name) { Some(v) => match v.clone().eval_to_const() { @@ -437,6 +439,7 @@ impl MagicAlgoApply { }, } } + #[allow(dead_code)] pub(crate) fn unit_interval_option(&self, name: &str, default: Option) -> Result { match self.options.get(name) { Some(v) => match v.clone().eval_to_const() { @@ -571,12 +574,14 @@ pub(crate) enum MagicAlgoRuleArg { } impl MagicAlgoRuleArg { + #[allow(dead_code)] pub(crate) fn bindings(&self) -> &[Symbol] { match self { MagicAlgoRuleArg::InMem { bindings, .. } | MagicAlgoRuleArg::Stored { bindings, .. } => bindings, } } + #[allow(dead_code)] pub(crate) fn span(&self) -> SourceSpan { match self { MagicAlgoRuleArg::InMem { span, .. } | MagicAlgoRuleArg::Stored { span, .. } => *span, diff --git a/cozo-core/src/data/tuple.rs b/cozo-core/src/data/tuple.rs index a538c22c..af7236d8 100644 --- a/cozo-core/src/data/tuple.rs +++ b/cozo-core/src/data/tuple.rs @@ -10,8 +10,6 @@ use crate::data::memcmp::MemCmpEncoder; use crate::data::value::DataValue; use crate::runtime::relation::RelationId; -pub(crate) const KEY_PREFIX_LEN: usize = 9; - #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Default)] pub struct Tuple(pub(crate) Vec); diff --git a/cozo-core/src/lib.rs b/cozo-core/src/lib.rs index 41c353a0..200765cd 100644 --- a/cozo-core/src/lib.rs +++ b/cozo-core/src/lib.rs @@ -23,7 +23,7 @@ //! We created an in-memory database with [`new_cozo_mem`](crate::new_cozo_mem) above. //! Persistent options include [`new_cozo_rocksdb`](crate::new_cozo_rocksdb), //! [`new_cozo_sqlite`](crate::new_cozo_sqlite) and others. - +#![doc = document_features::document_features!()] #![warn(rust_2018_idioms, future_incompatible)] #![warn(missing_docs)] #![allow(clippy::type_complexity)] @@ -34,9 +34,13 @@ pub use miette::Error; pub use runtime::db::Db; pub use runtime::relation::decode_tuple_from_kv; pub use storage::mem::{new_cozo_mem, MemStorage}; +#[cfg(feature = "storage-rocksdb")] pub use storage::rocks::{new_cozo_rocksdb, RocksDbStorage}; +#[cfg(feature = "storage-sled")] pub use storage::sled::{new_cozo_sled, SledStorage}; +#[cfg(feature = "storage-sqlite")] pub use storage::sqlite::{new_cozo_sqlite, SqliteStorage}; +#[cfg(feature = "storage-tikv")] pub use storage::tikv::{new_cozo_tikv, TiKvStorage}; pub use storage::{Storage, StoreTx}; diff --git a/cozo-core/src/runtime/db.rs b/cozo-core/src/runtime/db.rs index 706c8015..215d2ce6 100644 --- a/cozo-core/src/runtime/db.rs +++ b/cozo-core/src/runtime/db.rs @@ -59,9 +59,6 @@ pub struct DbManifest { pub storage_version: u64, } -// FIXME this should be storage-specific -pub(crate) const CURRENT_STORAGE_VERSION: u64 = 1; - /// The database object of Cozo. #[derive(Clone)] pub struct Db { diff --git a/cozo-core/src/storage/mod.rs b/cozo-core/src/storage/mod.rs index deef6d4c..aab2a605 100644 --- a/cozo-core/src/storage/mod.rs +++ b/cozo-core/src/storage/mod.rs @@ -9,9 +9,13 @@ use crate::data::tuple::Tuple; use crate::decode_tuple_from_kv; pub(crate) mod mem; +#[cfg(feature = "storage-rocksdb")] pub(crate) mod rocks; +#[cfg(feature = "storage-sled")] pub(crate) mod sled; +#[cfg(feature = "storage-sqlite")] pub(crate) mod sqlite; +#[cfg(feature = "storage-tikv")] pub(crate) mod tikv; // pub(crate) mod re; @@ -58,7 +62,7 @@ pub trait StoreTx<'s> { fn commit(&mut self) -> Result<()>; /// Scan on a range. `lower` is inclusive whereas `upper` is exclusive. - /// The default implementation calls [`range_scan_owned`](Self::range_scan_owned) and converts the results. + /// The default implementation calls [`range_scan_owned`](Self::range_scan) and converts the results. /// /// The implementation must call [`decode_tuple_from_kv`](crate::decode_tuple_from_kv) to obtain /// a decoded tuple in the loop of the iterator. diff --git a/cozo-core/src/storage/rocks.rs b/cozo-core/src/storage/rocks.rs index a25bbf4e..25186c55 100644 --- a/cozo-core/src/storage/rocks.rs +++ b/cozo-core/src/storage/rocks.rs @@ -9,13 +9,16 @@ use miette::{miette, IntoDiagnostic, Result, WrapErr}; use cozorocks::{DbBuilder, DbIter, RocksDb, Tx}; -use crate::data::tuple::{Tuple, KEY_PREFIX_LEN}; -use crate::runtime::db::{BadDbInit, DbManifest, CURRENT_STORAGE_VERSION}; +use crate::data::tuple::Tuple; +use crate::runtime::db::{BadDbInit, DbManifest}; use crate::runtime::relation::decode_tuple_from_kv; use crate::storage::{Storage, StoreTx}; use crate::utils::swap_option_result; use crate::Db; +const KEY_PREFIX_LEN: usize = 9; +const CURRENT_STORAGE_VERSION: u64 = 1; + /// Creates a RocksDB database object. /// This is currently the fastest persistent storage and it can /// sustain huge concurrency. diff --git a/cozo-core/tests/air_routes_mem.rs b/cozo-core/tests/air_routes_mem.rs index cf58ad79..834f2813 100644 --- a/cozo-core/tests/air_routes_mem.rs +++ b/cozo-core/tests/air_routes_mem.rs @@ -1,6 +1,7 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. */ +#![cfg(feature = "graph-algo")] use std::str::FromStr; use std::time::Instant; diff --git a/cozo-core/tests/air_routes_rocksdb.rs b/cozo-core/tests/air_routes_rocksdb.rs index 00b4a23b..bafc505e 100644 --- a/cozo-core/tests/air_routes_rocksdb.rs +++ b/cozo-core/tests/air_routes_rocksdb.rs @@ -1,6 +1,8 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. */ +#![cfg(feature = "storage-rocksdb")] +#![cfg(feature = "graph-algo")] use std::str::FromStr; use std::time::Instant; diff --git a/cozo-core/tests/air_routes_sled.rs b/cozo-core/tests/air_routes_sled.rs index bb05f279..42e947de 100644 --- a/cozo-core/tests/air_routes_sled.rs +++ b/cozo-core/tests/air_routes_sled.rs @@ -1,6 +1,8 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. */ +#![cfg(feature = "storage-sled")] +#![cfg(feature = "graph-algo")] use std::str::FromStr; use std::time::Instant; diff --git a/cozo-core/tests/air_routes_sqlite.rs b/cozo-core/tests/air_routes_sqlite.rs index 7d53c655..1d324423 100644 --- a/cozo-core/tests/air_routes_sqlite.rs +++ b/cozo-core/tests/air_routes_sqlite.rs @@ -1,6 +1,8 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. */ +#![cfg(feature = "storage-sqlite")] +#![cfg(feature = "graph-algo")] use std::str::FromStr; use std::time::Instant; diff --git a/cozo-core/tests/air_routes_tikv.rs b/cozo-core/tests/air_routes_tikv.rs index 94ad734b..c943be50 100644 --- a/cozo-core/tests/air_routes_tikv.rs +++ b/cozo-core/tests/air_routes_tikv.rs @@ -1,16 +1,19 @@ /* * Copyright 2022, The Cozo Project Authors. Licensed under AGPL-3 or later. */ +#![cfg(feature = "storage-tikv")] +#![cfg(feature = "graph-algo")] use std::str::FromStr; use std::time::Instant; use approx::AbsDiffEq; -use cozo::{new_cozo_tikv, Db, TiKvStorage}; use env_logger::Env; use lazy_static::lazy_static; use serde_json::json; +use cozo::{new_cozo_tikv, Db, TiKvStorage}; + lazy_static! { static ref TEST_DB: Db = { env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); diff --git a/cozoserver/Cargo.toml b/cozoserver/Cargo.toml new file mode 100644 index 00000000..d299f1a9 --- /dev/null +++ b/cozoserver/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "cozoserver" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cozo = { version = "0.1.2", path = "../cozo-core" } +clap = { version = "3.2.8", features = ["derive"] } +rouille = "3.5.0" diff --git a/cozo-core/src/bin/cozoserver.rs b/cozoserver/src/main.rs similarity index 100% rename from cozo-core/src/bin/cozoserver.rs rename to cozoserver/src/main.rs