From 659ee3bb2fdc6b4fe35a46fa6cab1d72a27025cf Mon Sep 17 00:00:00 2001 From: Ziyang Hu Date: Tue, 1 Nov 2022 16:51:34 +0800 Subject: [PATCH] nodejs module --- Cargo.lock | 88 ++++++++++++++++++++++++++++ Cargo.toml | 2 +- nodejs/.gitignore | 5 ++ nodejs/Cargo.toml | 24 ++++++++ nodejs/README.md | 121 +++++++++++++++++++++++++++++++++++++++ nodejs/package-lock.json | 34 +++++++++++ nodejs/package.json | 31 ++++++++++ nodejs/src/lib.rs | 114 ++++++++++++++++++++++++++++++++++++ python/src/lib.rs | 4 +- src/runtime/db.rs | 24 +++++--- 10 files changed, 437 insertions(+), 10 deletions(-) create mode 100644 nodejs/.gitignore create mode 100644 nodejs/Cargo.toml create mode 100644 nodejs/README.md create mode 100644 nodejs/package-lock.json create mode 100644 nodejs/package.json create mode 100644 nodejs/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 37ae9003..ebd874f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,6 +378,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "cozo-nodejs" +version = "0.1.1" +dependencies = [ + "cozo", + "lazy_static", + "miette", + "neon", + "serde_json", +] + [[package]] name = "cozo_py_module" version = "0.1.1" @@ -809,6 +820,16 @@ version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +[[package]] +name = "libloading" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] + [[package]] name = "link-cplusplus" version = "1.0.7" @@ -985,6 +1006,47 @@ dependencies = [ "syn", ] +[[package]] +name = "neon" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e15415261d880aed48122e917a45e87bb82cf0260bb6db48bbab44b7464373" +dependencies = [ + "neon-build", + "neon-macros", + "neon-runtime", + "semver", + "smallvec", +] + +[[package]] +name = "neon-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bac98a702e71804af3dacfde41edde4a16076a7bbe889ae61e56e18c5b1c811" + +[[package]] +name = "neon-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7288eac8b54af7913c60e0eb0e2a7683020dffa342ab3fd15e28f035ba897cf" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "neon-runtime" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676720fa8bb32c64c3d9f49c47a47289239ec46b4bdb66d0913cc512cb0daca" +dependencies = [ + "cfg-if 1.0.0", + "libloading", + "smallvec", +] + [[package]] name = "num-complex" version = "0.4.2" @@ -1570,6 +1632,21 @@ dependencies = [ "untrusted", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.147" @@ -1721,6 +1798,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "target-lexicon" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index 9808aa04..8b35fde8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,4 +68,4 @@ lto = true #debug = true [workspace] -members = ["cozorocks", "python"] +members = ["cozorocks", "python", "nodejs"] diff --git a/nodejs/.gitignore b/nodejs/.gitignore new file mode 100644 index 00000000..6ca71fb5 --- /dev/null +++ b/nodejs/.gitignore @@ -0,0 +1,5 @@ +target +index.node +**/node_modules +**/.DS_Store +npm-debug.log* diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml new file mode 100644 index 00000000..c12f7c06 --- /dev/null +++ b/nodejs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "cozo-nodejs" +version = "0.1.1" +description = "Cozo database for NodeJS" +authors = ["Ziyang Hu"] +license = "MIT/Apache-2.0/BSD-3-Clause" +edition = "2021" +exclude = ["index.node"] + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cozo = { version = "0.1.1", path = ".." } +miette = { version = "=5.3.0", features = ["fancy"] } +serde_json = "1.0.81" +lazy_static = "1.4.0" + +[dependencies.neon] +version = "0.10" +default-features = false +features = ["napi-6", "channel-api"] diff --git a/nodejs/README.md b/nodejs/README.md new file mode 100644 index 00000000..49e01817 --- /dev/null +++ b/nodejs/README.md @@ -0,0 +1,121 @@ +# cozo-nodejs + +**cozo-nodejs:** Cozo database for NodeJS + +This project was bootstrapped by [create-neon](https://www.npmjs.com/package/create-neon). + +## Installing cozo-nodejs + +Installing cozo-nodejs requires a [supported version of Node and Rust](https://github.com/neon-bindings/neon#platform-support). + +You can install the project with npm. In the project directory, run: + +```sh +$ npm install +``` + +This fully installs the project, including installing any dependencies and running the build. + +## Building cozo-nodejs + +If you have already installed the project and only want to run the build, run: + +```sh +$ npm run build +``` + +This command uses the [cargo-cp-artifact](https://github.com/neon-bindings/cargo-cp-artifact) utility to run the Rust build and copy the built library into `./index.node`. + +## Exploring cozo-nodejs + +After building cozo-nodejs, you can explore its exports at the Node REPL: + +```sh +$ npm install +$ node +> require('.').hello() +"hello node" +``` + +## Available Scripts + +In the project directory, you can run: + +### `npm install` + +Installs the project, including running `npm run build`. + +### `npm build` + +Builds the Node addon (`index.node`) from source. + +Additional [`cargo build`](https://doc.rust-lang.org/cargo/commands/cargo-build.html) arguments may be passed to `npm build` and `npm build-*` commands. For example, to enable a [cargo feature](https://doc.rust-lang.org/cargo/reference/features.html): + +``` +npm run build -- --feature=beetle +``` + +#### `npm build-debug` + +Alias for `npm build`. + +#### `npm build-release` + +Same as [`npm build`](#npm-build) but, builds the module with the [`release`](https://doc.rust-lang.org/cargo/reference/profiles.html#release) profile. Release builds will compile slower, but run faster. + +### `npm test` + +Runs the unit tests by calling `cargo test`. You can learn more about [adding tests to your Rust code](https://doc.rust-lang.org/book/ch11-01-writing-tests.html) from the [Rust book](https://doc.rust-lang.org/book/). + +## Project Layout + +The directory structure of this project is: + +``` +cozo-nodejs/ +├── Cargo.toml +├── README.md +├── index.node +├── package.json +├── src/ +| └── lib.rs +└── target/ +``` + +### Cargo.toml + +The Cargo [manifest file](https://doc.rust-lang.org/cargo/reference/manifest.html), which informs the `cargo` command. + +### README.md + +This file. + +### index.node + +The Node addon—i.e., a binary Node module—generated by building the project. This is the main module for this package, as dictated by the `"main"` key in `package.json`. + +Under the hood, a [Node addon](https://nodejs.org/api/addons.html) is a [dynamically-linked shared object](https://en.wikipedia.org/wiki/Library_(computing)#Shared_libraries). The `"build"` script produces this file by copying it from within the `target/` directory, which is where the Rust build produces the shared object. + +### package.json + +The npm [manifest file](https://docs.npmjs.com/cli/v7/configuring-npm/package-json), which informs the `npm` command. + +### src/ + +The directory tree containing the Rust source code for the project. + +### src/lib.rs + +The Rust library's main module. + +### target/ + +Binary artifacts generated by the Rust build. + +## Learn More + +To learn more about Neon, see the [Neon documentation](https://neon-bindings.com). + +To learn more about Rust, see the [Rust documentation](https://www.rust-lang.org). + +To learn more about Node, see the [Node documentation](https://nodejs.org). diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json new file mode 100644 index 00000000..bbe517ec --- /dev/null +++ b/nodejs/package-lock.json @@ -0,0 +1,34 @@ +{ + "name": "cozo-nodejs", + "version": "0.1.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "cozo-nodejs", + "version": "0.1.1", + "hasInstallScript": true, + "license": "MIT", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } + }, + "node_modules/cargo-cp-artifact": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/cargo-cp-artifact/-/cargo-cp-artifact-0.1.6.tgz", + "integrity": "sha512-CQw0doK/aaF7j041666XzuilHxqMxaKkn+I5vmBsd8SAwS0cO5CqVEVp0xJwOKstyqWZ6WK4Ww3O6p26x/Goyg==", + "dev": true, + "bin": { + "cargo-cp-artifact": "bin/cargo-cp-artifact.js" + } + } + }, + "dependencies": { + "cargo-cp-artifact": { + "version": "0.1.6", + "resolved": "https://registry.npmmirror.com/cargo-cp-artifact/-/cargo-cp-artifact-0.1.6.tgz", + "integrity": "sha512-CQw0doK/aaF7j041666XzuilHxqMxaKkn+I5vmBsd8SAwS0cO5CqVEVp0xJwOKstyqWZ6WK4Ww3O6p26x/Goyg==", + "dev": true + } + } +} diff --git a/nodejs/package.json b/nodejs/package.json new file mode 100644 index 00000000..e6650ff8 --- /dev/null +++ b/nodejs/package.json @@ -0,0 +1,31 @@ +{ + "name": "cozo-nodejs", + "version": "0.1.1", + "description": "Cozo database for NodeJS", + "main": "index.node", + "scripts": { + "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build-debug": "npm run build --", + "build-release": "npm run build -- --release", + "install": "npm run build-release", + "test": "cargo test" + }, + "author": "Ziyang Hu", + "license": "MIT", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/cozodb/cozo.git" + }, + "keywords": [ + "database", + "datalog", + "graph" + ], + "bugs": { + "url": "https://github.com/cozodb/cozo/issues" + }, + "homepage": "https://github.com/cozodb/cozo#readme" +} \ No newline at end of file diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs new file mode 100644 index 00000000..037d5daa --- /dev/null +++ b/nodejs/src/lib.rs @@ -0,0 +1,114 @@ +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Mutex; + +use lazy_static::lazy_static; +use neon::prelude::*; + +use cozo::Db; + +#[derive(Default)] +struct Handles { + current: AtomicU32, + dbs: Mutex>, +} + +lazy_static! { + static ref HANDLES: Handles = Handles::default(); +} + +fn open_db(mut cx: FunctionContext) -> JsResult { + let path = cx.argument::(0)?.value(&mut cx); + match Db::new(path) { + Ok(db) => { + let id = HANDLES.current.fetch_add(1, Ordering::AcqRel); + let mut dbs = HANDLES.dbs.lock().unwrap(); + dbs.insert(id, db); + Ok(cx.number(id)) + } + Err(err) => { + let s = cx.string(format!("{:?}", err)); + cx.throw(s) + } + } +} + +fn close_db(mut cx: FunctionContext) -> JsResult { + let id = cx.argument::(0)?.value(&mut cx) as u32; + let db = { + let mut dbs = HANDLES.dbs.lock().unwrap(); + dbs.remove(&id) + }; + Ok(cx.boolean(db.is_some())) +} + +fn query_db(mut cx: FunctionContext) -> JsResult { + let id = cx.argument::(0)?.value(&mut cx) as u32; + let db = { + let db_ref = { + let dbs = HANDLES.dbs.lock().unwrap(); + dbs.get(&id).cloned() + }; + match db_ref { + None => { + let s = cx.string("database already closed"); + cx.throw(s)? + } + Some(db) => db, + } + }; + + let query = cx.argument::(1)?.value(&mut cx); + let params_str = cx.argument::(2)?.value(&mut cx); + + let params_map: serde_json::Value = match serde_json::from_str(¶ms_str) { + Ok(m) => m, + Err(_) => { + let s = cx.string("the given params argument is not valid JSON"); + cx.throw(s)? + } + }; + let params_arg: BTreeMap<_, _> = match params_map { + serde_json::Value::Object(m) => m.into_iter().collect(), + _ => { + let s = cx.string("the given params argument is not a JSON map"); + cx.throw(s)? + } + }; + + let callback = cx.argument::(3)?.root(&mut cx); + + let channel = cx.channel(); + + std::thread::spawn(move || { + let result = db.run_script(&query, ¶ms_arg); + channel.send(move |mut cx| { + let callback = callback.into_inner(&mut cx); + let this = cx.undefined(); + let args = match result { + Ok(json) => { + let json_str = cx.string(json.to_string()); + vec![cx.null().upcast::(), json_str.upcast()] + } + Err(err) => { + let err = cx.string(format!("{:?}", err)); + vec![err.upcast::()] + } + }; + + callback.call(&mut cx, this, args)?; + + Ok(()) + }); + }); + + Ok(cx.undefined()) +} + +#[neon::main] +fn main(mut cx: ModuleContext) -> NeonResult<()> { + cx.export_function("open_db", open_db)?; + cx.export_function("close_db", close_db)?; + cx.export_function("query_db", query_db)?; + Ok(()) +} diff --git a/python/src/lib.rs b/python/src/lib.rs index f79a82dd..c0a54fce 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -37,11 +37,11 @@ impl CozoDbPy { } pub fn run_query(&self, py: Python<'_>, query: &str, params: &str) -> PyResult { let params_map: serde_json::Value = serde_json::from_str(params) - .map_err(|_| miette!("the given params is not valid JSON")) + .map_err(|_| miette!("the given params argument is not valid JSON")) .into_py_res()?; let params_arg: BTreeMap<_, _> = match params_map { serde_json::Value::Object(m) => m.into_iter().collect(), - _ => Err(miette!("the given params is not a map")).into_py_res()?, + _ => Err(miette!("the given params argument is not a JSON map")).into_py_res()?, }; let ret = py.allow_threads(|| self.db.run_script(query, ¶ms_arg).into_py_res())?; Ok(ret.to_string()) diff --git a/src/runtime/db.rs b/src/runtime/db.rs index 5c4ee00b..29c9ef21 100644 --- a/src/runtime/db.rs +++ b/src/runtime/db.rs @@ -12,7 +12,7 @@ use std::{fs, thread}; use either::{Left, Right}; use itertools::Itertools; -use miette::{bail, ensure, Diagnostic, Result, WrapErr}; +use miette::{bail, ensure, miette, Diagnostic, IntoDiagnostic, Result, WrapErr}; use serde_json::json; use smartstring::SmartString; use thiserror::Error; @@ -60,6 +60,7 @@ pub(crate) struct DbManifest { const CURRENT_STORAGE_VERSION: u64 = 1; /// The database object of Cozo. +#[derive(Clone)] pub struct Db { db: RocksDb, relation_store_id: Arc, @@ -93,9 +94,12 @@ impl Db { if manifest_path.exists() { let existing: DbManifest = rmp_serde::from_slice( - &fs::read(manifest_path).expect("reading manifest failed"), + &fs::read(manifest_path) + .into_diagnostic() + .wrap_err_with(|| "when reading manifest")?, ) - .expect("parsing manifest failed"); + .into_diagnostic() + .wrap_err_with(|| "when reading manifest")?; assert_eq!( existing.storage_version, CURRENT_STORAGE_VERSION, "Unknown storage version {}", @@ -108,9 +112,11 @@ impl Db { rmp_serde::to_vec_named(&DbManifest { storage_version: CURRENT_STORAGE_VERSION, }) - .expect("serializing manifest failed"), + .into_diagnostic() + .wrap_err_with(|| "when serializing manifest")?, ) - .expect("Writing to manifest failed"); + .into_diagnostic() + .wrap_err_with(|| "when serializing manifest")?; true } }; @@ -121,7 +127,11 @@ impl Db { .create_if_missing(is_new) .use_capped_prefix_extractor(true, KEY_PREFIX_LEN) .use_bloom_filter(true, 9.9, true) - .path(store_path.to_str().unwrap()); + .path( + store_path + .to_str() + .ok_or_else(|| miette!("bad path name"))?, + ); let db = db_builder.build()?; @@ -521,7 +531,7 @@ impl Db { let now = SystemTime::now(); let since_the_epoch = now .duration_since(UNIX_EPOCH) - .expect("Time went backwards") + .into_diagnostic()? .as_secs_f64(); let handle = RunningQueryHandle {